@drarzter/kafka-client 0.9.4 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +693 -8
- package/dist/chunk-OR7TPAAE.mjs +4760 -0
- package/dist/chunk-OR7TPAAE.mjs.map +1 -0
- package/dist/chunk-PQVBRDNV.mjs +149 -0
- package/dist/chunk-PQVBRDNV.mjs.map +1 -0
- package/dist/cli/dlq.d.ts +119 -0
- package/dist/cli/dlq.d.ts.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/{chunk-SM4FZKAZ.mjs → cli/index.js} +1073 -309
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/index.mjs +356 -0
- package/dist/cli/index.mjs.map +1 -0
- package/dist/client/config/from-env.d.ts +188 -0
- package/dist/client/config/from-env.d.ts.map +1 -0
- package/dist/client/config/index.d.ts +2 -0
- package/dist/client/config/index.d.ts.map +1 -0
- package/dist/client/errors.d.ts +67 -0
- package/dist/client/errors.d.ts.map +1 -0
- package/dist/client/kafka.client/admin/ops.d.ts +114 -0
- package/dist/client/kafka.client/admin/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts +24 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +52 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts +4 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts +10 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts +5 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/handler.d.ts +163 -0
- package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/ops.d.ts +64 -0
- package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts +168 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/queue.d.ts +37 -0
- package/dist/client/kafka.client/consumer/queue.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts +68 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/setup.d.ts +66 -0
- package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/start.d.ts +7 -0
- package/dist/client/kafka.client/consumer/start.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/stop.d.ts +19 -0
- package/dist/client/kafka.client/consumer/stop.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts +4 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts.map +1 -0
- package/dist/client/kafka.client/context.d.ts +75 -0
- package/dist/client/kafka.client/context.d.ts.map +1 -0
- package/dist/client/kafka.client/index.d.ts +155 -0
- package/dist/client/kafka.client/index.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts +61 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts +28 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts +22 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts +67 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts +41 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/ops.d.ts +79 -0
- package/dist/client/kafka.client/producer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/send.d.ts +21 -0
- package/dist/client/kafka.client/producer/send.d.ts.map +1 -0
- package/dist/client/kafka.client/validate-options.d.ts +11 -0
- package/dist/client/kafka.client/validate-options.d.ts.map +1 -0
- package/dist/client/message/envelope.d.ts +105 -0
- package/dist/client/message/envelope.d.ts.map +1 -0
- package/dist/client/message/schema-registry.d.ts +124 -0
- package/dist/client/message/schema-registry.d.ts.map +1 -0
- package/dist/client/message/serde.d.ts +68 -0
- package/dist/client/message/serde.d.ts.map +1 -0
- package/dist/client/message/topic.d.ts +159 -0
- package/dist/client/message/topic.d.ts.map +1 -0
- package/dist/client/message/versioned-schema.d.ts +53 -0
- package/dist/client/message/versioned-schema.d.ts.map +1 -0
- package/dist/client/outbox/index.d.ts +4 -0
- package/dist/client/outbox/index.d.ts.map +1 -0
- package/dist/client/outbox/outbox.relay.d.ts +90 -0
- package/dist/client/outbox/outbox.relay.d.ts.map +1 -0
- package/dist/client/outbox/outbox.store.d.ts +42 -0
- package/dist/client/outbox/outbox.store.d.ts.map +1 -0
- package/dist/client/outbox/outbox.types.d.ts +144 -0
- package/dist/client/outbox/outbox.types.d.ts.map +1 -0
- package/dist/client/security/acl.d.ts +108 -0
- package/dist/client/security/acl.d.ts.map +1 -0
- package/dist/client/security/index.d.ts +5 -0
- package/dist/client/security/index.d.ts.map +1 -0
- package/dist/client/security/providers.d.ts +88 -0
- package/dist/client/security/providers.d.ts.map +1 -0
- package/dist/client/security/resolve-security.d.ts +19 -0
- package/dist/client/security/resolve-security.d.ts.map +1 -0
- package/dist/client/security/security.types.d.ts +76 -0
- package/dist/client/security/security.types.d.ts.map +1 -0
- package/dist/client/transport/confluent.transport.d.ts +32 -0
- package/dist/client/transport/confluent.transport.d.ts.map +1 -0
- package/dist/client/transport/transport.interface.d.ts +221 -0
- package/dist/client/transport/transport.interface.d.ts.map +1 -0
- package/dist/client/types/admin.interface.d.ts +174 -0
- package/dist/client/types/admin.interface.d.ts.map +1 -0
- package/dist/client/types/admin.types.d.ts +140 -0
- package/dist/client/types/admin.types.d.ts.map +1 -0
- package/dist/client/types/client.d.ts +21 -0
- package/dist/client/types/client.d.ts.map +1 -0
- package/dist/client/types/common.d.ts +84 -0
- package/dist/client/types/common.d.ts.map +1 -0
- package/dist/client/types/config.types.d.ts +167 -0
- package/dist/client/types/config.types.d.ts.map +1 -0
- package/dist/client/types/consumer.interface.d.ts +115 -0
- package/dist/client/types/consumer.interface.d.ts.map +1 -0
- package/dist/{consumer.types-fFCag3VJ.d.mts → client/types/consumer.types.d.ts} +62 -383
- package/dist/client/types/consumer.types.d.ts.map +1 -0
- package/dist/client/types/dedup.types.d.ts +50 -0
- package/dist/client/types/dedup.types.d.ts.map +1 -0
- package/dist/client/types/lifecycle.interface.d.ts +72 -0
- package/dist/client/types/lifecycle.interface.d.ts.map +1 -0
- package/dist/client/types/producer.interface.d.ts +52 -0
- package/dist/client/types/producer.interface.d.ts.map +1 -0
- package/dist/client/types/producer.types.d.ts +90 -0
- package/dist/client/types/producer.types.d.ts.map +1 -0
- package/dist/client/types.d.ts +8 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/core.d.ts +13 -314
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +1466 -123
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +45 -3
- package/dist/index.d.ts +7 -128
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1483 -123
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +62 -3
- package/dist/index.mjs.map +1 -1
- package/dist/nest/kafka.constants.d.ts +5 -0
- package/dist/nest/kafka.constants.d.ts.map +1 -0
- package/dist/nest/kafka.decorator.d.ts +49 -0
- package/dist/nest/kafka.decorator.d.ts.map +1 -0
- package/dist/nest/kafka.explorer.d.ts +17 -0
- package/dist/nest/kafka.explorer.d.ts.map +1 -0
- package/dist/nest/kafka.health.d.ts +7 -0
- package/dist/nest/kafka.health.d.ts.map +1 -0
- package/dist/nest/kafka.module.d.ts +61 -0
- package/dist/nest/kafka.module.d.ts.map +1 -0
- package/dist/otel.d.ts +83 -5
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +100 -6
- package/dist/otel.js.map +1 -1
- package/dist/otel.mjs +98 -5
- package/dist/otel.mjs.map +1 -1
- package/dist/serde.d.ts +157 -0
- package/dist/serde.d.ts.map +1 -0
- package/dist/serde.js +308 -0
- package/dist/serde.js.map +1 -0
- package/dist/serde.mjs +158 -0
- package/dist/serde.mjs.map +1 -0
- package/dist/testing/client.mock.d.ts +47 -0
- package/dist/testing/client.mock.d.ts.map +1 -0
- package/dist/testing/index.d.ts +4 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/test.container.d.ts +63 -0
- package/dist/testing/test.container.d.ts.map +1 -0
- package/dist/{testing.d.mts → testing/transport.fake.d.ts} +7 -111
- package/dist/testing/transport.fake.d.ts.map +1 -0
- package/dist/testing.d.ts +2 -318
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +26 -0
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +26 -0
- package/dist/testing.mjs.map +1 -1
- package/package.json +40 -8
- package/dist/chunk-SM4FZKAZ.mjs.map +0 -1
- package/dist/client-1irhGEu0.d.mts +0 -751
- package/dist/client-BpFjkHhr.d.ts +0 -751
- package/dist/consumer.types-fFCag3VJ.d.ts +0 -958
- package/dist/core.d.mts +0 -314
- package/dist/index.d.mts +0 -128
- package/dist/otel.d.mts +0 -27
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
|
|
4
|
+
// src/client/transport/confluent.transport.ts
|
|
5
|
+
var import_kafka_javascript = require("@confluentinc/kafka-javascript");
|
|
6
|
+
var { Kafka: KafkaClass, logLevel: KafkaLogLevel, PartitionAssigners } = import_kafka_javascript.KafkaJS;
|
|
4
7
|
var ConfluentTransaction = class {
|
|
5
8
|
constructor(tx) {
|
|
6
9
|
this.tx = tx;
|
|
7
10
|
}
|
|
11
|
+
tx;
|
|
8
12
|
async send(record) {
|
|
9
13
|
await this.tx.send(record);
|
|
10
14
|
}
|
|
@@ -26,10 +30,17 @@ var ConfluentProducer = class {
|
|
|
26
30
|
constructor(producer) {
|
|
27
31
|
this.producer = producer;
|
|
28
32
|
}
|
|
33
|
+
producer;
|
|
34
|
+
connectPromise;
|
|
29
35
|
async connect() {
|
|
30
|
-
|
|
36
|
+
this.connectPromise ??= this.producer.connect().catch((err) => {
|
|
37
|
+
this.connectPromise = void 0;
|
|
38
|
+
throw err;
|
|
39
|
+
});
|
|
40
|
+
return this.connectPromise;
|
|
31
41
|
}
|
|
32
42
|
async disconnect() {
|
|
43
|
+
this.connectPromise = void 0;
|
|
33
44
|
await this.producer.disconnect();
|
|
34
45
|
}
|
|
35
46
|
async send(record) {
|
|
@@ -44,6 +55,7 @@ var ConfluentConsumer = class {
|
|
|
44
55
|
constructor(consumer) {
|
|
45
56
|
this.consumer = consumer;
|
|
46
57
|
}
|
|
58
|
+
consumer;
|
|
47
59
|
/** Returns the underlying KafkaJS.Consumer — used by ConfluentTransaction.sendOffsets. */
|
|
48
60
|
getNative() {
|
|
49
61
|
return this.consumer;
|
|
@@ -83,6 +95,7 @@ var ConfluentAdmin = class {
|
|
|
83
95
|
constructor(admin) {
|
|
84
96
|
this.admin = admin;
|
|
85
97
|
}
|
|
98
|
+
admin;
|
|
86
99
|
async connect() {
|
|
87
100
|
await this.admin.connect();
|
|
88
101
|
}
|
|
@@ -92,11 +105,11 @@ var ConfluentAdmin = class {
|
|
|
92
105
|
async createTopics(options) {
|
|
93
106
|
await this.admin.createTopics(options);
|
|
94
107
|
}
|
|
95
|
-
async fetchTopicOffsets(
|
|
96
|
-
return this.admin.fetchTopicOffsets(
|
|
108
|
+
async fetchTopicOffsets(topic) {
|
|
109
|
+
return this.admin.fetchTopicOffsets(topic);
|
|
97
110
|
}
|
|
98
|
-
async fetchTopicOffsetsByTimestamp(
|
|
99
|
-
return this.admin.
|
|
111
|
+
async fetchTopicOffsetsByTimestamp(topic, timestamp) {
|
|
112
|
+
return this.admin.fetchTopicOffsetsByTimestamp(topic, timestamp);
|
|
100
113
|
}
|
|
101
114
|
async fetchOffsets(options) {
|
|
102
115
|
return this.admin.fetchOffsets(options);
|
|
@@ -122,10 +135,29 @@ var ConfluentAdmin = class {
|
|
|
122
135
|
};
|
|
123
136
|
var ConfluentTransport = class {
|
|
124
137
|
kafka;
|
|
125
|
-
constructor(clientId, brokers) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
138
|
+
constructor(clientId, brokers, security) {
|
|
139
|
+
const kafkaJS = { clientId, brokers, logLevel: KafkaLogLevel.ERROR };
|
|
140
|
+
if (security?.ssl !== void 0) kafkaJS.ssl = security.ssl;
|
|
141
|
+
if (security?.sasl) {
|
|
142
|
+
if (security.sasl.mechanism === "oauthbearer") {
|
|
143
|
+
const provider = security.sasl.oauthBearerProvider;
|
|
144
|
+
kafkaJS.sasl = {
|
|
145
|
+
mechanism: "oauthbearer",
|
|
146
|
+
oauthBearerProvider: async () => {
|
|
147
|
+
const token = await provider();
|
|
148
|
+
return {
|
|
149
|
+
value: token.value,
|
|
150
|
+
principal: token.principal ?? "kafka-client",
|
|
151
|
+
lifetime: token.lifetimeMs ?? Date.now() + 15 * 6e4,
|
|
152
|
+
...token.extensions && { extensions: token.extensions }
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
} else {
|
|
157
|
+
kafkaJS.sasl = security.sasl;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
this.kafka = new KafkaClass({ kafkaJS });
|
|
129
161
|
}
|
|
130
162
|
producer(options) {
|
|
131
163
|
const native = this.kafka.producer({
|
|
@@ -152,6 +184,9 @@ var ConfluentTransport = class {
|
|
|
152
184
|
partitionAssigners: [assigner]
|
|
153
185
|
}
|
|
154
186
|
};
|
|
187
|
+
if (options.groupInstanceId) {
|
|
188
|
+
config["group.instance.id"] = options.groupInstanceId;
|
|
189
|
+
}
|
|
155
190
|
if (options.onRebalance) {
|
|
156
191
|
const cb = options.onRebalance;
|
|
157
192
|
config.rebalance_cb = (err, assignment) => {
|
|
@@ -169,16 +204,49 @@ var ConfluentTransport = class {
|
|
|
169
204
|
}
|
|
170
205
|
};
|
|
171
206
|
|
|
207
|
+
// src/client/message/serde.ts
|
|
208
|
+
var JsonSerde = class {
|
|
209
|
+
/** JSON-stringify the validated payload. Returns a UTF-8 string. */
|
|
210
|
+
serialize(value) {
|
|
211
|
+
return JSON.stringify(value);
|
|
212
|
+
}
|
|
213
|
+
/** JSON-parse UTF-8 wire bytes into an object. */
|
|
214
|
+
deserialize(data) {
|
|
215
|
+
return JSON.parse(data.toString("utf8"));
|
|
216
|
+
}
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
// src/client/kafka.client/infra/dedup.store.ts
|
|
220
|
+
var InMemoryDedupStore = class {
|
|
221
|
+
constructor(states) {
|
|
222
|
+
this.states = states;
|
|
223
|
+
}
|
|
224
|
+
states;
|
|
225
|
+
getLastClock(groupId, topicPartition) {
|
|
226
|
+
return this.states.get(groupId)?.get(topicPartition);
|
|
227
|
+
}
|
|
228
|
+
setLastClock(groupId, topicPartition, clock) {
|
|
229
|
+
let group = this.states.get(groupId);
|
|
230
|
+
if (!group) {
|
|
231
|
+
group = /* @__PURE__ */ new Map();
|
|
232
|
+
this.states.set(groupId, group);
|
|
233
|
+
}
|
|
234
|
+
group.set(topicPartition, clock);
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
|
|
172
238
|
// src/client/message/envelope.ts
|
|
173
|
-
|
|
174
|
-
|
|
239
|
+
var import_node_async_hooks = require("async_hooks");
|
|
240
|
+
var import_node_crypto = require("crypto");
|
|
175
241
|
var HEADER_EVENT_ID = "x-event-id";
|
|
176
242
|
var HEADER_CORRELATION_ID = "x-correlation-id";
|
|
177
243
|
var HEADER_TIMESTAMP = "x-timestamp";
|
|
178
244
|
var HEADER_SCHEMA_VERSION = "x-schema-version";
|
|
179
245
|
var HEADER_TRACEPARENT = "traceparent";
|
|
180
246
|
var HEADER_LAMPORT_CLOCK = "x-lamport-clock";
|
|
181
|
-
var
|
|
247
|
+
var HEADER_DELAYED_UNTIL = "x-delayed-until";
|
|
248
|
+
var HEADER_DELAYED_TARGET = "x-delayed-target";
|
|
249
|
+
var envelopeStorage = new import_node_async_hooks.AsyncLocalStorage();
|
|
182
250
|
function getEnvelopeContext() {
|
|
183
251
|
return envelopeStorage.getStore();
|
|
184
252
|
}
|
|
@@ -187,8 +255,8 @@ function runWithEnvelopeContext(ctx, fn) {
|
|
|
187
255
|
}
|
|
188
256
|
function buildEnvelopeHeaders(options = {}) {
|
|
189
257
|
const ctx = getEnvelopeContext();
|
|
190
|
-
const correlationId = options.correlationId ?? ctx?.correlationId ?? randomUUID();
|
|
191
|
-
const eventId = options.eventId ?? randomUUID();
|
|
258
|
+
const correlationId = options.correlationId ?? ctx?.correlationId ?? (0, import_node_crypto.randomUUID)();
|
|
259
|
+
const eventId = options.eventId ?? (0, import_node_crypto.randomUUID)();
|
|
192
260
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
193
261
|
const schemaVersion = String(options.schemaVersion ?? 1);
|
|
194
262
|
const envelope = {
|
|
@@ -216,14 +284,14 @@ function decodeHeaders(raw) {
|
|
|
216
284
|
}
|
|
217
285
|
return result;
|
|
218
286
|
}
|
|
219
|
-
function extractEnvelope(payload, headers,
|
|
287
|
+
function extractEnvelope(payload, headers, topic, partition, offset) {
|
|
220
288
|
return {
|
|
221
289
|
payload,
|
|
222
|
-
topic
|
|
290
|
+
topic,
|
|
223
291
|
partition,
|
|
224
292
|
offset,
|
|
225
|
-
eventId: headers[HEADER_EVENT_ID] ?? randomUUID(),
|
|
226
|
-
correlationId: headers[HEADER_CORRELATION_ID] ?? randomUUID(),
|
|
293
|
+
eventId: headers[HEADER_EVENT_ID] ?? (0, import_node_crypto.randomUUID)(),
|
|
294
|
+
correlationId: headers[HEADER_CORRELATION_ID] ?? (0, import_node_crypto.randomUUID)(),
|
|
227
295
|
timestamp: headers[HEADER_TIMESTAMP] ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
228
296
|
schemaVersion: Number(headers[HEADER_SCHEMA_VERSION] ?? 1),
|
|
229
297
|
traceparent: headers[HEADER_TRACEPARENT],
|
|
@@ -232,38 +300,49 @@ function extractEnvelope(payload, headers, topic2, partition, offset) {
|
|
|
232
300
|
}
|
|
233
301
|
|
|
234
302
|
// src/client/errors.ts
|
|
303
|
+
function toError(error) {
|
|
304
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
305
|
+
}
|
|
235
306
|
var KafkaProcessingError = class extends Error {
|
|
236
|
-
constructor(message,
|
|
307
|
+
constructor(message, topic, originalMessage, options) {
|
|
237
308
|
super(message, options);
|
|
238
|
-
this.topic =
|
|
309
|
+
this.topic = topic;
|
|
239
310
|
this.originalMessage = originalMessage;
|
|
240
311
|
this.name = "KafkaProcessingError";
|
|
241
312
|
if (options?.cause) this.cause = options.cause;
|
|
242
313
|
}
|
|
314
|
+
topic;
|
|
315
|
+
originalMessage;
|
|
243
316
|
};
|
|
244
317
|
var KafkaValidationError = class extends Error {
|
|
245
|
-
constructor(
|
|
246
|
-
super(`Schema validation failed for topic "${
|
|
247
|
-
this.topic =
|
|
318
|
+
constructor(topic, originalMessage, options) {
|
|
319
|
+
super(`Schema validation failed for topic "${topic}"`, options);
|
|
320
|
+
this.topic = topic;
|
|
248
321
|
this.originalMessage = originalMessage;
|
|
249
322
|
this.name = "KafkaValidationError";
|
|
250
323
|
if (options?.cause) this.cause = options.cause;
|
|
251
324
|
}
|
|
325
|
+
topic;
|
|
326
|
+
originalMessage;
|
|
252
327
|
};
|
|
253
328
|
var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
254
|
-
constructor(
|
|
329
|
+
constructor(topic, originalMessage, attempts, options) {
|
|
255
330
|
super(
|
|
256
|
-
`Message processing failed after ${attempts} attempts on topic "${
|
|
257
|
-
|
|
331
|
+
`Message processing failed after ${attempts} attempts on topic "${topic}"`,
|
|
332
|
+
topic,
|
|
258
333
|
originalMessage,
|
|
259
334
|
options
|
|
260
335
|
);
|
|
261
336
|
this.attempts = attempts;
|
|
262
337
|
this.name = "KafkaRetryExhaustedError";
|
|
263
338
|
}
|
|
339
|
+
attempts;
|
|
264
340
|
};
|
|
265
341
|
|
|
266
342
|
// src/client/kafka.client/producer/ops.ts
|
|
343
|
+
function resolveSerde(topicOrDesc, clientSerde) {
|
|
344
|
+
return topicOrDesc?.__serde ?? clientSerde;
|
|
345
|
+
}
|
|
267
346
|
function resolveTopicName(topicOrDescriptor) {
|
|
268
347
|
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
269
348
|
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
@@ -273,14 +352,14 @@ function resolveTopicName(topicOrDescriptor) {
|
|
|
273
352
|
}
|
|
274
353
|
function registerSchema(topicOrDesc, schemaRegistry, logger) {
|
|
275
354
|
if (topicOrDesc?.__schema) {
|
|
276
|
-
const
|
|
277
|
-
const existing = schemaRegistry.get(
|
|
355
|
+
const topic = resolveTopicName(topicOrDesc);
|
|
356
|
+
const existing = schemaRegistry.get(topic);
|
|
278
357
|
if (existing && existing !== topicOrDesc.__schema) {
|
|
279
358
|
logger?.warn(
|
|
280
|
-
`Schema conflict for topic "${
|
|
359
|
+
`Schema conflict for topic "${topic}": a different schema is already registered. Using the new schema \u2014 ensure consistent schemas to avoid silent validation mismatches.`
|
|
281
360
|
);
|
|
282
361
|
}
|
|
283
|
-
schemaRegistry.set(
|
|
362
|
+
schemaRegistry.set(topic, topicOrDesc.__schema);
|
|
284
363
|
}
|
|
285
364
|
}
|
|
286
365
|
async function validateMessage(topicOrDesc, message, deps, ctx) {
|
|
@@ -309,7 +388,8 @@ async function validateMessage(topicOrDesc, message, deps, ctx) {
|
|
|
309
388
|
return message;
|
|
310
389
|
}
|
|
311
390
|
async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
312
|
-
const
|
|
391
|
+
const topic = resolveTopicName(topicOrDesc);
|
|
392
|
+
const serde = resolveSerde(topicOrDesc, deps.serde);
|
|
313
393
|
const builtMessages = await Promise.all(
|
|
314
394
|
messages.map(async (m) => {
|
|
315
395
|
const envelopeHeaders = buildEnvelopeHeaders({
|
|
@@ -322,27 +402,32 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
322
402
|
envelopeHeaders[HEADER_LAMPORT_CLOCK] = String(deps.nextLamportClock());
|
|
323
403
|
}
|
|
324
404
|
for (const inst of deps.instrumentation) {
|
|
325
|
-
inst.beforeSend?.(
|
|
405
|
+
inst.beforeSend?.(topic, envelopeHeaders);
|
|
326
406
|
}
|
|
327
407
|
const sendCtx = {
|
|
328
|
-
topic
|
|
408
|
+
topic,
|
|
329
409
|
headers: envelopeHeaders,
|
|
330
410
|
version: m.schemaVersion ?? 1
|
|
331
411
|
};
|
|
412
|
+
const validated = await validateMessage(topicOrDesc, m.value, deps, sendCtx);
|
|
332
413
|
return {
|
|
333
|
-
value:
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
414
|
+
value: await serde.serialize(validated, {
|
|
415
|
+
topic,
|
|
416
|
+
headers: envelopeHeaders,
|
|
417
|
+
isKey: false
|
|
418
|
+
}),
|
|
419
|
+
// Explicit key wins; otherwise fall back to the descriptor's .key()
|
|
420
|
+
// extractor (runs on the original, pre-validation payload).
|
|
421
|
+
key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
|
|
337
422
|
headers: envelopeHeaders
|
|
338
423
|
};
|
|
339
424
|
})
|
|
340
425
|
);
|
|
341
|
-
return { topic
|
|
426
|
+
return { topic, messages: builtMessages, ...compression && { compression } };
|
|
342
427
|
}
|
|
343
428
|
|
|
344
429
|
// src/client/kafka.client/consumer/ops.ts
|
|
345
|
-
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment) {
|
|
430
|
+
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment, groupInstanceId) {
|
|
346
431
|
const { consumers, consumerCreationOptions, transport, onRebalance, logger } = deps;
|
|
347
432
|
if (consumers.has(groupId)) {
|
|
348
433
|
const prev = consumerCreationOptions.get(groupId);
|
|
@@ -375,6 +460,7 @@ function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partition
|
|
|
375
460
|
fromBeginning,
|
|
376
461
|
autoCommit,
|
|
377
462
|
partitionAssigner: partitionAssigner ?? "cooperative-sticky",
|
|
463
|
+
groupInstanceId,
|
|
378
464
|
onRebalance: (type, assignments) => {
|
|
379
465
|
if (type === "assign") fireOnAssignment();
|
|
380
466
|
else if (type === "revoke") scheduleSettle();
|
|
@@ -414,12 +500,22 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
|
|
|
414
500
|
}
|
|
415
501
|
return schemaMap;
|
|
416
502
|
}
|
|
503
|
+
function buildSerdeMap(topics) {
|
|
504
|
+
let serdeMap;
|
|
505
|
+
for (const t of topics) {
|
|
506
|
+
if (t?.__serde) {
|
|
507
|
+
(serdeMap ??= /* @__PURE__ */ new Map()).set(resolveTopicName(t), t.__serde);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return serdeMap;
|
|
511
|
+
}
|
|
417
512
|
|
|
418
513
|
// src/client/kafka.client/admin/ops.ts
|
|
419
514
|
var AdminOps = class {
|
|
420
515
|
constructor(deps) {
|
|
421
516
|
this.deps = deps;
|
|
422
517
|
}
|
|
518
|
+
deps;
|
|
423
519
|
isConnected = false;
|
|
424
520
|
/** Underlying admin client — used by index.ts for topic validation. */
|
|
425
521
|
get admin() {
|
|
@@ -450,7 +546,7 @@ var AdminOps = class {
|
|
|
450
546
|
await this.deps.admin.disconnect();
|
|
451
547
|
this.isConnected = false;
|
|
452
548
|
}
|
|
453
|
-
async resetOffsets(groupId,
|
|
549
|
+
async resetOffsets(groupId, topic, position) {
|
|
454
550
|
const gid = groupId ?? this.deps.defaultGroupId;
|
|
455
551
|
if (this.deps.runningConsumers.has(gid)) {
|
|
456
552
|
throw new Error(
|
|
@@ -458,14 +554,14 @@ var AdminOps = class {
|
|
|
458
554
|
);
|
|
459
555
|
}
|
|
460
556
|
await this.ensureConnected();
|
|
461
|
-
const partitionOffsets = await this.deps.admin.fetchTopicOffsets(
|
|
557
|
+
const partitionOffsets = await this.deps.admin.fetchTopicOffsets(topic);
|
|
462
558
|
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
463
559
|
partition,
|
|
464
560
|
offset: position === "earliest" ? low : high
|
|
465
561
|
}));
|
|
466
|
-
await this.deps.admin.setOffsets({ groupId: gid, topic
|
|
562
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic, partitions });
|
|
467
563
|
this.deps.logger.log(
|
|
468
|
-
`Offsets reset to ${position} for group "${gid}" on topic "${
|
|
564
|
+
`Offsets reset to ${position} for group "${gid}" on topic "${topic}"`
|
|
469
565
|
);
|
|
470
566
|
}
|
|
471
567
|
/**
|
|
@@ -482,15 +578,15 @@ var AdminOps = class {
|
|
|
482
578
|
}
|
|
483
579
|
await this.ensureConnected();
|
|
484
580
|
const byTopic = /* @__PURE__ */ new Map();
|
|
485
|
-
for (const { topic
|
|
486
|
-
const list = byTopic.get(
|
|
581
|
+
for (const { topic, partition, offset } of assignments) {
|
|
582
|
+
const list = byTopic.get(topic) ?? [];
|
|
487
583
|
list.push({ partition, offset });
|
|
488
|
-
byTopic.set(
|
|
584
|
+
byTopic.set(topic, list);
|
|
489
585
|
}
|
|
490
|
-
for (const [
|
|
491
|
-
await this.deps.admin.setOffsets({ groupId: gid, topic
|
|
586
|
+
for (const [topic, partitions] of byTopic) {
|
|
587
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic, partitions });
|
|
492
588
|
this.deps.logger.log(
|
|
493
|
-
`Offsets set for group "${gid}" on "${
|
|
589
|
+
`Offsets set for group "${gid}" on "${topic}": ${JSON.stringify(partitions)}`
|
|
494
590
|
);
|
|
495
591
|
}
|
|
496
592
|
}
|
|
@@ -511,27 +607,30 @@ var AdminOps = class {
|
|
|
511
607
|
}
|
|
512
608
|
await this.ensureConnected();
|
|
513
609
|
const byTopic = /* @__PURE__ */ new Map();
|
|
514
|
-
for (const { topic
|
|
515
|
-
const list = byTopic.get(
|
|
610
|
+
for (const { topic, partition, timestamp } of assignments) {
|
|
611
|
+
const list = byTopic.get(topic) ?? [];
|
|
516
612
|
list.push({ partition, timestamp });
|
|
517
|
-
byTopic.set(
|
|
613
|
+
byTopic.set(topic, list);
|
|
518
614
|
}
|
|
519
|
-
for (const [
|
|
615
|
+
for (const [topic, parts] of byTopic) {
|
|
520
616
|
const offsets = await Promise.all(
|
|
521
617
|
parts.map(async ({ partition, timestamp }) => {
|
|
522
618
|
const results = await this.deps.admin.fetchTopicOffsetsByTimestamp(
|
|
523
|
-
|
|
619
|
+
topic,
|
|
524
620
|
timestamp
|
|
525
621
|
);
|
|
526
622
|
const found = results.find(
|
|
527
623
|
(r) => r.partition === partition
|
|
528
624
|
);
|
|
529
|
-
return { partition, offset: found
|
|
625
|
+
if (found) return { partition, offset: found.offset };
|
|
626
|
+
const topicOffsets = await this.deps.admin.fetchTopicOffsets(topic);
|
|
627
|
+
const po = topicOffsets.find((o) => o.partition === partition);
|
|
628
|
+
return { partition, offset: po?.high ?? "0" };
|
|
530
629
|
})
|
|
531
630
|
);
|
|
532
|
-
await this.deps.admin.setOffsets({ groupId: gid, topic
|
|
631
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic, partitions: offsets });
|
|
533
632
|
this.deps.logger.log(
|
|
534
|
-
`Offsets set by timestamp for group "${gid}" on "${
|
|
633
|
+
`Offsets set by timestamp for group "${gid}" on "${topic}": ${JSON.stringify(offsets)}`
|
|
535
634
|
);
|
|
536
635
|
}
|
|
537
636
|
}
|
|
@@ -551,11 +650,11 @@ var AdminOps = class {
|
|
|
551
650
|
await this.ensureConnected();
|
|
552
651
|
const committedByTopic = await this.deps.admin.fetchOffsets({ groupId: gid });
|
|
553
652
|
const brokerOffsetsAll = await Promise.all(
|
|
554
|
-
committedByTopic.map(({ topic
|
|
653
|
+
committedByTopic.map(({ topic }) => this.deps.admin.fetchTopicOffsets(topic))
|
|
555
654
|
);
|
|
556
655
|
const result = [];
|
|
557
656
|
for (let i = 0; i < committedByTopic.length; i++) {
|
|
558
|
-
const { topic
|
|
657
|
+
const { topic, partitions } = committedByTopic[i];
|
|
559
658
|
const brokerOffsets = brokerOffsetsAll[i];
|
|
560
659
|
for (const { partition, offset } of partitions) {
|
|
561
660
|
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
@@ -563,7 +662,7 @@ var AdminOps = class {
|
|
|
563
662
|
const committed = parseInt(offset, 10);
|
|
564
663
|
const high = parseInt(broker.high, 10);
|
|
565
664
|
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
566
|
-
result.push({ topic
|
|
665
|
+
result.push({ topic, partition, lag });
|
|
567
666
|
}
|
|
568
667
|
}
|
|
569
668
|
return result;
|
|
@@ -607,7 +706,8 @@ var AdminOps = class {
|
|
|
607
706
|
name: t.name,
|
|
608
707
|
partitions: t.partitions.map((p) => ({
|
|
609
708
|
partition: p.partitionId ?? p.partition ?? 0,
|
|
610
|
-
|
|
709
|
+
// -1 is Kafka's own "no leader" sentinel; 0 is a valid broker id
|
|
710
|
+
leader: p.leader ?? -1,
|
|
611
711
|
replicas: (p.replicas ?? []).map(
|
|
612
712
|
(r) => typeof r === "number" ? r : r.nodeId
|
|
613
713
|
),
|
|
@@ -632,9 +732,9 @@ var AdminOps = class {
|
|
|
632
732
|
* Delete records from a topic up to (but not including) the given offsets.
|
|
633
733
|
* All messages with offsets **before** the given offset are deleted.
|
|
634
734
|
*/
|
|
635
|
-
async deleteRecords(
|
|
735
|
+
async deleteRecords(topic, partitions) {
|
|
636
736
|
await this.ensureConnected();
|
|
637
|
-
await this.deps.admin.deleteTopicRecords({ topic
|
|
737
|
+
await this.deps.admin.deleteTopicRecords({ topic, partitions });
|
|
638
738
|
}
|
|
639
739
|
/**
|
|
640
740
|
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
@@ -691,28 +791,14 @@ var AdminOps = class {
|
|
|
691
791
|
};
|
|
692
792
|
|
|
693
793
|
// src/client/kafka.client/consumer/pipeline.ts
|
|
694
|
-
function toError(error) {
|
|
695
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
696
|
-
}
|
|
697
794
|
function sleep(ms) {
|
|
698
795
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
699
796
|
}
|
|
700
|
-
function
|
|
701
|
-
|
|
702
|
-
return JSON.parse(raw);
|
|
703
|
-
} catch (error) {
|
|
704
|
-
logger.error(
|
|
705
|
-
`Failed to parse message from topic ${topic2}:`,
|
|
706
|
-
toError(error).stack
|
|
707
|
-
);
|
|
708
|
-
return null;
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
|
-
async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
|
|
712
|
-
const schema = schemaMap.get(topic2);
|
|
797
|
+
async function validateWithSchema(message, raw, topic, schemaMap, interceptors, dlq, deps) {
|
|
798
|
+
const schema = schemaMap.get(topic);
|
|
713
799
|
if (!schema) return message;
|
|
714
800
|
const ctx = {
|
|
715
|
-
topic
|
|
801
|
+
topic,
|
|
716
802
|
headers: deps.originalHeaders ?? {},
|
|
717
803
|
version: Number(deps.originalHeaders?.["x-schema-version"] ?? 1)
|
|
718
804
|
};
|
|
@@ -720,22 +806,22 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
|
|
|
720
806
|
return await schema.parse(message, ctx);
|
|
721
807
|
} catch (error) {
|
|
722
808
|
const err = toError(error);
|
|
723
|
-
const validationError = new KafkaValidationError(
|
|
809
|
+
const validationError = new KafkaValidationError(topic, message, {
|
|
724
810
|
cause: err
|
|
725
811
|
});
|
|
726
812
|
deps.logger.error(
|
|
727
|
-
`Schema validation failed for topic ${
|
|
813
|
+
`Schema validation failed for topic ${topic}:`,
|
|
728
814
|
err.message
|
|
729
815
|
);
|
|
730
816
|
if (dlq) {
|
|
731
|
-
await sendToDlq(
|
|
817
|
+
await sendToDlq(topic, raw, deps, {
|
|
732
818
|
error: validationError,
|
|
733
819
|
attempt: 0,
|
|
734
820
|
originalHeaders: deps.originalHeaders
|
|
735
821
|
});
|
|
736
822
|
} else {
|
|
737
823
|
await deps.onMessageLost?.({
|
|
738
|
-
topic
|
|
824
|
+
topic,
|
|
739
825
|
error: validationError,
|
|
740
826
|
attempt: 0,
|
|
741
827
|
headers: deps.originalHeaders ?? {}
|
|
@@ -744,7 +830,7 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
|
|
|
744
830
|
const errorEnvelope = extractEnvelope(
|
|
745
831
|
message,
|
|
746
832
|
deps.originalHeaders ?? {},
|
|
747
|
-
|
|
833
|
+
topic,
|
|
748
834
|
-1,
|
|
749
835
|
""
|
|
750
836
|
);
|
|
@@ -757,11 +843,11 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
|
|
|
757
843
|
return null;
|
|
758
844
|
}
|
|
759
845
|
}
|
|
760
|
-
function buildDlqPayload(
|
|
761
|
-
const dlqTopic = `${
|
|
846
|
+
function buildDlqPayload(topic, rawMessage, meta) {
|
|
847
|
+
const dlqTopic = `${topic}.dlq`;
|
|
762
848
|
const headers = {
|
|
763
849
|
...meta?.originalHeaders ?? {},
|
|
764
|
-
"x-dlq-original-topic":
|
|
850
|
+
"x-dlq-original-topic": topic,
|
|
765
851
|
"x-dlq-failed-at": (/* @__PURE__ */ new Date()).toISOString(),
|
|
766
852
|
"x-dlq-error-message": meta?.error.message ?? "unknown",
|
|
767
853
|
"x-dlq-error-stack": meta?.error.stack?.slice(0, 2e3) ?? "",
|
|
@@ -769,8 +855,8 @@ function buildDlqPayload(topic2, rawMessage, meta) {
|
|
|
769
855
|
};
|
|
770
856
|
return { topic: dlqTopic, messages: [{ value: rawMessage, headers }] };
|
|
771
857
|
}
|
|
772
|
-
async function sendToDlq(
|
|
773
|
-
const payload = buildDlqPayload(
|
|
858
|
+
async function sendToDlq(topic, rawMessage, deps, meta) {
|
|
859
|
+
const payload = buildDlqPayload(topic, rawMessage, meta);
|
|
774
860
|
try {
|
|
775
861
|
await deps.producer.send(payload);
|
|
776
862
|
deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
|
|
@@ -781,7 +867,7 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
|
|
|
781
867
|
err.stack
|
|
782
868
|
);
|
|
783
869
|
await deps.onMessageLost?.({
|
|
784
|
-
topic
|
|
870
|
+
topic,
|
|
785
871
|
error: err,
|
|
786
872
|
attempt: meta?.attempt ?? 0,
|
|
787
873
|
headers: meta?.originalHeaders ?? {}
|
|
@@ -954,7 +1040,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
954
1040
|
const backoffMs = retry?.backoffMs ?? 1e3;
|
|
955
1041
|
const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
|
|
956
1042
|
const envelopes = Array.isArray(envelope) ? envelope : [envelope];
|
|
957
|
-
const
|
|
1043
|
+
const topic = envelopes[0]?.topic ?? "unknown";
|
|
958
1044
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
959
1045
|
const error = await runHandlerWithPipeline(
|
|
960
1046
|
fn,
|
|
@@ -977,16 +1063,17 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
977
1063
|
for (const env of envelopes) deps.onMessage?.(env);
|
|
978
1064
|
return;
|
|
979
1065
|
}
|
|
1066
|
+
deps.onFailure?.(envelopes[0]);
|
|
980
1067
|
const isLastAttempt = attempt === maxAttempts;
|
|
981
1068
|
const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
|
|
982
|
-
|
|
1069
|
+
topic,
|
|
983
1070
|
envelopes.map((e) => e.payload),
|
|
984
1071
|
maxAttempts,
|
|
985
1072
|
{ cause: error }
|
|
986
1073
|
) : error;
|
|
987
1074
|
await notifyInterceptorsOnError(envelopes, interceptors, reportedError);
|
|
988
1075
|
deps.logger.error(
|
|
989
|
-
`Error processing ${isBatch ? "batch" : "message"} from topic ${
|
|
1076
|
+
`Error processing ${isBatch ? "batch" : "message"} from topic ${topic} (attempt ${attempt}/${maxAttempts}):`,
|
|
990
1077
|
error.stack
|
|
991
1078
|
);
|
|
992
1079
|
if (retryTopics && retry) {
|
|
@@ -1004,7 +1091,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
1004
1091
|
}
|
|
1005
1092
|
} else {
|
|
1006
1093
|
await sendToRetryTopic(
|
|
1007
|
-
|
|
1094
|
+
topic,
|
|
1008
1095
|
rawMessages,
|
|
1009
1096
|
1,
|
|
1010
1097
|
retry.maxRetries,
|
|
@@ -1017,7 +1104,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
1017
1104
|
} else if (isLastAttempt) {
|
|
1018
1105
|
if (dlq) {
|
|
1019
1106
|
for (let i = 0; i < rawMessages.length; i++) {
|
|
1020
|
-
await sendToDlq(
|
|
1107
|
+
await sendToDlq(topic, rawMessages[i], deps, {
|
|
1021
1108
|
error,
|
|
1022
1109
|
attempt,
|
|
1023
1110
|
originalHeaders: envelopes[i]?.headers
|
|
@@ -1026,7 +1113,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
1026
1113
|
}
|
|
1027
1114
|
} else {
|
|
1028
1115
|
await deps.onMessageLost?.({
|
|
1029
|
-
topic
|
|
1116
|
+
topic,
|
|
1030
1117
|
error,
|
|
1031
1118
|
attempt,
|
|
1032
1119
|
headers: envelopes[0]?.headers ?? {}
|
|
@@ -1061,9 +1148,14 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
|
1061
1148
|
}
|
|
1062
1149
|
}
|
|
1063
1150
|
|
|
1064
|
-
// src/client/kafka.client/consumer/dlq-replay.ts
|
|
1065
|
-
async function replayDlqTopic(
|
|
1066
|
-
|
|
1151
|
+
// src/client/kafka.client/consumer/features/dlq-replay.ts
|
|
1152
|
+
async function replayDlqTopic(topic, deps, options = {}) {
|
|
1153
|
+
if (topic.endsWith(".dlq")) {
|
|
1154
|
+
throw new Error(
|
|
1155
|
+
`replayDlq: pass the ORIGINAL topic name \u2014 "${topic}" already ends in ".dlq" (the ".dlq" suffix is appended internally, so this would read "${topic}.dlq")`
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
1158
|
+
const dlqTopic = `${topic}.dlq`;
|
|
1067
1159
|
const partitionOffsets = await deps.fetchTopicOffsets(dlqTopic);
|
|
1068
1160
|
const activePartitions = partitionOffsets.filter(
|
|
1069
1161
|
(p) => Number.parseInt(p.high, 10) > Number.parseInt(p.low, 10)
|
|
@@ -1094,15 +1186,15 @@ async function replayDlqTopic(topic2, deps, options = {}) {
|
|
|
1094
1186
|
const originalHeaders = Object.fromEntries(
|
|
1095
1187
|
Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
|
|
1096
1188
|
);
|
|
1097
|
-
const
|
|
1098
|
-
const shouldProcess = !options.filter || options.filter(headers,
|
|
1189
|
+
const bytes = message.value;
|
|
1190
|
+
const shouldProcess = !options.filter || options.filter(headers, bytes.toString("utf8"));
|
|
1099
1191
|
if (!targetTopic || !shouldProcess) {
|
|
1100
1192
|
skipped++;
|
|
1101
1193
|
} else if (options.dryRun) {
|
|
1102
1194
|
deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
|
|
1103
1195
|
replayed++;
|
|
1104
1196
|
} else {
|
|
1105
|
-
await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
|
|
1197
|
+
await deps.send(targetTopic, [{ value: bytes, headers: originalHeaders }]);
|
|
1106
1198
|
replayed++;
|
|
1107
1199
|
}
|
|
1108
1200
|
const allDone = Array.from(highWatermarks.entries()).every(
|
|
@@ -1128,19 +1220,20 @@ var MetricsManager = class {
|
|
|
1128
1220
|
constructor(deps) {
|
|
1129
1221
|
this.deps = deps;
|
|
1130
1222
|
}
|
|
1223
|
+
deps;
|
|
1131
1224
|
topicMetrics = /* @__PURE__ */ new Map();
|
|
1132
|
-
metricsFor(
|
|
1133
|
-
let m = this.topicMetrics.get(
|
|
1225
|
+
metricsFor(topic) {
|
|
1226
|
+
let m = this.topicMetrics.get(topic);
|
|
1134
1227
|
if (!m) {
|
|
1135
1228
|
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1136
|
-
this.topicMetrics.set(
|
|
1229
|
+
this.topicMetrics.set(topic, m);
|
|
1137
1230
|
}
|
|
1138
1231
|
return m;
|
|
1139
1232
|
}
|
|
1140
1233
|
/** Fire `afterSend` instrumentation hooks for each message in a batch. */
|
|
1141
|
-
notifyAfterSend(
|
|
1234
|
+
notifyAfterSend(topic, count) {
|
|
1142
1235
|
for (let i = 0; i < count; i++)
|
|
1143
|
-
for (const inst of this.deps.instrumentation) inst.afterSend?.(
|
|
1236
|
+
for (const inst of this.deps.instrumentation) inst.afterSend?.(topic);
|
|
1144
1237
|
}
|
|
1145
1238
|
/**
|
|
1146
1239
|
* Increment the retry counter for the envelope's topic and fire all `onRetry` instrumentation hooks.
|
|
@@ -1153,16 +1246,25 @@ var MetricsManager = class {
|
|
|
1153
1246
|
for (const inst of this.deps.instrumentation) inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1154
1247
|
}
|
|
1155
1248
|
/**
|
|
1156
|
-
* Increment the DLQ counter for the envelope's topic
|
|
1157
|
-
*
|
|
1249
|
+
* Increment the DLQ counter for the envelope's topic and fire all `onDlq` instrumentation hooks.
|
|
1250
|
+
* Circuit breaker failures are recorded separately via `notifyFailure` at the
|
|
1251
|
+
* handler-error boundary — dead-lettering itself is not a circuit event.
|
|
1158
1252
|
* @param envelope The message envelope being sent to the DLQ.
|
|
1159
1253
|
* @param reason The reason the message is being dead-lettered.
|
|
1160
|
-
* @param gid Consumer group ID — used to drive circuit breaker state.
|
|
1161
1254
|
*/
|
|
1162
|
-
notifyDlq(envelope, reason
|
|
1255
|
+
notifyDlq(envelope, reason) {
|
|
1163
1256
|
this.metricsFor(envelope.topic).dlqCount++;
|
|
1164
1257
|
for (const inst of this.deps.instrumentation) inst.onDlq?.(envelope, reason);
|
|
1165
|
-
|
|
1258
|
+
}
|
|
1259
|
+
/**
|
|
1260
|
+
* Notify the circuit breaker of a handler failure. Fired on every failed
|
|
1261
|
+
* handler attempt (in-process retries and retry-topic levels included),
|
|
1262
|
+
* independent of whether the message is ultimately dead-lettered.
|
|
1263
|
+
* @param envelope The message envelope whose handler failed.
|
|
1264
|
+
* @param gid Consumer group ID — used to drive circuit breaker state.
|
|
1265
|
+
*/
|
|
1266
|
+
notifyFailure(envelope, gid) {
|
|
1267
|
+
this.deps.onCircuitFailure(envelope, gid);
|
|
1166
1268
|
}
|
|
1167
1269
|
/**
|
|
1168
1270
|
* Increment the deduplication counter for the envelope's topic and fire all `onDuplicate` hooks.
|
|
@@ -1189,9 +1291,9 @@ var MetricsManager = class {
|
|
|
1189
1291
|
* @param topic When provided, returns counters for that topic only; otherwise aggregates all topics.
|
|
1190
1292
|
* @returns Read-only `KafkaMetrics` snapshot. Returns zero-valued counters if the topic has no events.
|
|
1191
1293
|
*/
|
|
1192
|
-
getMetrics(
|
|
1193
|
-
if (
|
|
1194
|
-
const m = this.topicMetrics.get(
|
|
1294
|
+
getMetrics(topic) {
|
|
1295
|
+
if (topic !== void 0) {
|
|
1296
|
+
const m = this.topicMetrics.get(topic);
|
|
1195
1297
|
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1196
1298
|
}
|
|
1197
1299
|
const agg = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
@@ -1207,9 +1309,9 @@ var MetricsManager = class {
|
|
|
1207
1309
|
* Reset event counters to zero.
|
|
1208
1310
|
* @param topic When provided, clears counters for that topic only; otherwise clears all topics.
|
|
1209
1311
|
*/
|
|
1210
|
-
resetMetrics(
|
|
1211
|
-
if (
|
|
1212
|
-
this.topicMetrics.delete(
|
|
1312
|
+
resetMetrics(topic) {
|
|
1313
|
+
if (topic !== void 0) {
|
|
1314
|
+
this.topicMetrics.delete(topic);
|
|
1213
1315
|
return;
|
|
1214
1316
|
}
|
|
1215
1317
|
this.topicMetrics.clear();
|
|
@@ -1221,6 +1323,7 @@ var InFlightTracker = class {
|
|
|
1221
1323
|
constructor(warn) {
|
|
1222
1324
|
this.warn = warn;
|
|
1223
1325
|
}
|
|
1326
|
+
warn;
|
|
1224
1327
|
inFlightTotal = 0;
|
|
1225
1328
|
drainResolvers = [];
|
|
1226
1329
|
/**
|
|
@@ -1231,10 +1334,16 @@ var InFlightTracker = class {
|
|
|
1231
1334
|
*/
|
|
1232
1335
|
track(fn) {
|
|
1233
1336
|
this.inFlightTotal++;
|
|
1234
|
-
|
|
1337
|
+
const done = () => {
|
|
1235
1338
|
this.inFlightTotal--;
|
|
1236
1339
|
if (this.inFlightTotal === 0) this.drainResolvers.splice(0).forEach((r) => r());
|
|
1237
|
-
}
|
|
1340
|
+
};
|
|
1341
|
+
try {
|
|
1342
|
+
return fn().finally(done);
|
|
1343
|
+
} catch (err) {
|
|
1344
|
+
done();
|
|
1345
|
+
throw err;
|
|
1346
|
+
}
|
|
1238
1347
|
}
|
|
1239
1348
|
/**
|
|
1240
1349
|
* Resolve when all tracked handlers have completed, or after `timeoutMs` elapses.
|
|
@@ -1268,6 +1377,7 @@ var CircuitBreakerManager = class {
|
|
|
1268
1377
|
constructor(deps) {
|
|
1269
1378
|
this.deps = deps;
|
|
1270
1379
|
}
|
|
1380
|
+
deps;
|
|
1271
1381
|
states = /* @__PURE__ */ new Map();
|
|
1272
1382
|
configs = /* @__PURE__ */ new Map();
|
|
1273
1383
|
/**
|
|
@@ -1283,8 +1393,8 @@ var CircuitBreakerManager = class {
|
|
|
1283
1393
|
* Returns a snapshot of the circuit breaker state for a given topic-partition.
|
|
1284
1394
|
* Returns `undefined` when no state exists for the key.
|
|
1285
1395
|
*/
|
|
1286
|
-
getState(
|
|
1287
|
-
const state = this.states.get(`${gid}:${
|
|
1396
|
+
getState(topic, partition, gid) {
|
|
1397
|
+
const state = this.states.get(`${gid}:${topic}:${partition}`);
|
|
1288
1398
|
if (!state) return void 0;
|
|
1289
1399
|
return {
|
|
1290
1400
|
status: state.status,
|
|
@@ -1412,6 +1522,9 @@ var AsyncQueue = class {
|
|
|
1412
1522
|
this.onFull = onFull;
|
|
1413
1523
|
this.onDrained = onDrained;
|
|
1414
1524
|
}
|
|
1525
|
+
highWaterMark;
|
|
1526
|
+
onFull;
|
|
1527
|
+
onDrained;
|
|
1415
1528
|
items = [];
|
|
1416
1529
|
waiting = [];
|
|
1417
1530
|
closed = false;
|
|
@@ -1423,6 +1536,7 @@ var AsyncQueue = class {
|
|
|
1423
1536
|
* @param item The value to enqueue.
|
|
1424
1537
|
*/
|
|
1425
1538
|
push(item) {
|
|
1539
|
+
if (this.closed) return;
|
|
1426
1540
|
if (this.waiting.length > 0) {
|
|
1427
1541
|
this.waiting.shift().resolve({ value: item, done: false });
|
|
1428
1542
|
} else {
|
|
@@ -1473,20 +1587,115 @@ var AsyncQueue = class {
|
|
|
1473
1587
|
}
|
|
1474
1588
|
};
|
|
1475
1589
|
|
|
1590
|
+
// src/client/kafka.client/validate-options.ts
|
|
1591
|
+
function validateClientOptions(clientId, groupId, brokers, options) {
|
|
1592
|
+
const problems = [];
|
|
1593
|
+
if (typeof clientId !== "string" || clientId.trim() === "") {
|
|
1594
|
+
problems.push("clientId must be a non-empty string");
|
|
1595
|
+
}
|
|
1596
|
+
if (typeof groupId !== "string" || groupId.trim() === "") {
|
|
1597
|
+
problems.push("groupId must be a non-empty string");
|
|
1598
|
+
}
|
|
1599
|
+
if (!Array.isArray(brokers) || brokers.length === 0 && !options?.transport) {
|
|
1600
|
+
problems.push("brokers must be a non-empty array of broker addresses");
|
|
1601
|
+
} else if (brokers.some((b) => typeof b !== "string" || b.trim() === "")) {
|
|
1602
|
+
problems.push("brokers must not contain empty entries");
|
|
1603
|
+
}
|
|
1604
|
+
if (options) {
|
|
1605
|
+
const {
|
|
1606
|
+
numPartitions,
|
|
1607
|
+
transactionalId,
|
|
1608
|
+
clockRecovery,
|
|
1609
|
+
lagThrottle
|
|
1610
|
+
} = options;
|
|
1611
|
+
if (numPartitions !== void 0 && (!Number.isInteger(numPartitions) || numPartitions < 1)) {
|
|
1612
|
+
problems.push(
|
|
1613
|
+
`numPartitions must be a positive integer (got ${numPartitions})`
|
|
1614
|
+
);
|
|
1615
|
+
}
|
|
1616
|
+
if (transactionalId !== void 0 && transactionalId.trim() === "") {
|
|
1617
|
+
problems.push("transactionalId must be a non-empty string when set");
|
|
1618
|
+
}
|
|
1619
|
+
if (clockRecovery) {
|
|
1620
|
+
if (!Array.isArray(clockRecovery.topics)) {
|
|
1621
|
+
problems.push("clockRecovery.topics must be an array of topic names");
|
|
1622
|
+
}
|
|
1623
|
+
if (clockRecovery.timeoutMs !== void 0 && !(clockRecovery.timeoutMs > 0)) {
|
|
1624
|
+
problems.push(
|
|
1625
|
+
`clockRecovery.timeoutMs must be > 0 (got ${clockRecovery.timeoutMs})`
|
|
1626
|
+
);
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
if (lagThrottle) {
|
|
1630
|
+
if (!(lagThrottle.maxLag >= 0)) {
|
|
1631
|
+
problems.push(`lagThrottle.maxLag must be >= 0 (got ${lagThrottle.maxLag})`);
|
|
1632
|
+
}
|
|
1633
|
+
if (lagThrottle.pollIntervalMs !== void 0 && !(lagThrottle.pollIntervalMs > 0)) {
|
|
1634
|
+
problems.push(
|
|
1635
|
+
`lagThrottle.pollIntervalMs must be > 0 (got ${lagThrottle.pollIntervalMs})`
|
|
1636
|
+
);
|
|
1637
|
+
}
|
|
1638
|
+
if (lagThrottle.maxWaitMs !== void 0 && !(lagThrottle.maxWaitMs >= 0)) {
|
|
1639
|
+
problems.push(
|
|
1640
|
+
`lagThrottle.maxWaitMs must be >= 0 (got ${lagThrottle.maxWaitMs})`
|
|
1641
|
+
);
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1645
|
+
if (problems.length > 0) {
|
|
1646
|
+
throw new Error(
|
|
1647
|
+
`KafkaClient: invalid configuration:
|
|
1648
|
+
- ${problems.join("\n- ")}`
|
|
1649
|
+
);
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
// src/client/security/resolve-security.ts
|
|
1654
|
+
var LOCAL_HOST_PATTERNS = [
|
|
1655
|
+
/^localhost(:\d+)?$/i,
|
|
1656
|
+
/^127\.\d+\.\d+\.\d+(:\d+)?$/,
|
|
1657
|
+
/^\[?::1\]?(:\d+)?$/,
|
|
1658
|
+
/^0\.0\.0\.0(:\d+)?$/,
|
|
1659
|
+
/^host\.docker\.internal(:\d+)?$/i
|
|
1660
|
+
];
|
|
1661
|
+
function isLocalBroker(broker) {
|
|
1662
|
+
return LOCAL_HOST_PATTERNS.some((re) => re.test(broker.trim()));
|
|
1663
|
+
}
|
|
1664
|
+
function resolveSecurityOptions(security, brokers, logger) {
|
|
1665
|
+
const hasRemoteBroker = brokers.some((b) => !isLocalBroker(b));
|
|
1666
|
+
if (!security?.sasl && security?.ssl !== true) {
|
|
1667
|
+
if (hasRemoteBroker && !security?.allowInsecure) {
|
|
1668
|
+
logger.warn(
|
|
1669
|
+
"Connecting to non-local brokers without TLS or SASL \u2014 traffic and payloads travel in plaintext. Configure `security: { ssl, sasl }` for production clusters, or set `security: { allowInsecure: true }` to acknowledge an intentionally insecure setup."
|
|
1670
|
+
);
|
|
1671
|
+
}
|
|
1672
|
+
return security;
|
|
1673
|
+
}
|
|
1674
|
+
if (security.sasl && security.ssl === void 0) {
|
|
1675
|
+
return { ...security, ssl: true };
|
|
1676
|
+
}
|
|
1677
|
+
if (security.sasl && security.ssl === false) {
|
|
1678
|
+
logger.warn(
|
|
1679
|
+
"SASL credentials are configured with `ssl: false` \u2014 credentials will be sent over plaintext. This is only safe on fully trusted networks."
|
|
1680
|
+
);
|
|
1681
|
+
}
|
|
1682
|
+
return security;
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1476
1685
|
// src/client/kafka.client/producer/lifecycle.ts
|
|
1477
1686
|
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1478
|
-
async function ensureTopic(ctx,
|
|
1479
|
-
if (!ctx.autoCreateTopicsEnabled || ctx.ensuredTopics.has(
|
|
1480
|
-
let p = ctx.ensureTopicPromises.get(
|
|
1687
|
+
async function ensureTopic(ctx, topic) {
|
|
1688
|
+
if (!ctx.autoCreateTopicsEnabled || ctx.ensuredTopics.has(topic)) return;
|
|
1689
|
+
let p = ctx.ensureTopicPromises.get(topic);
|
|
1481
1690
|
if (!p) {
|
|
1482
1691
|
p = (async () => {
|
|
1483
1692
|
await ctx.adminOps.ensureConnected();
|
|
1484
1693
|
await ctx.adminOps.admin.createTopics({
|
|
1485
|
-
topics: [{ topic
|
|
1694
|
+
topics: [{ topic, numPartitions: ctx.numPartitions }]
|
|
1486
1695
|
});
|
|
1487
|
-
ctx.ensuredTopics.add(
|
|
1488
|
-
})().finally(() => ctx.ensureTopicPromises.delete(
|
|
1489
|
-
ctx.ensureTopicPromises.set(
|
|
1696
|
+
ctx.ensuredTopics.add(topic);
|
|
1697
|
+
})().finally(() => ctx.ensureTopicPromises.delete(topic));
|
|
1698
|
+
ctx.ensureTopicPromises.set(topic, p);
|
|
1490
1699
|
}
|
|
1491
1700
|
await p;
|
|
1492
1701
|
}
|
|
@@ -1606,6 +1815,7 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1606
1815
|
const remaining = new Set(
|
|
1607
1816
|
partitionsToRead.map((p) => `${p.topic}:${p.partition}`)
|
|
1608
1817
|
);
|
|
1818
|
+
let settled = false;
|
|
1609
1819
|
const cleanup = () => {
|
|
1610
1820
|
consumer.disconnect().catch(() => {
|
|
1611
1821
|
}).finally(() => {
|
|
@@ -1613,6 +1823,16 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1613
1823
|
});
|
|
1614
1824
|
});
|
|
1615
1825
|
};
|
|
1826
|
+
const timeoutTimer = setTimeout(() => {
|
|
1827
|
+
if (settled) return;
|
|
1828
|
+
settled = true;
|
|
1829
|
+
ctx.logger.warn(
|
|
1830
|
+
`Clock recovery: timed out after ${ctx.clockRecoveryTimeoutMs} ms with ${remaining.size} partition(s) unread \u2014 proceeding with partial result`
|
|
1831
|
+
);
|
|
1832
|
+
cleanup();
|
|
1833
|
+
resolve();
|
|
1834
|
+
}, ctx.clockRecoveryTimeoutMs);
|
|
1835
|
+
timeoutTimer.unref?.();
|
|
1616
1836
|
consumer.connect().then(async () => {
|
|
1617
1837
|
const uniqueTopics = [
|
|
1618
1838
|
...new Set(partitionsToRead.map((p) => p.topic))
|
|
@@ -1633,13 +1853,18 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1633
1853
|
const clock = Number(raw);
|
|
1634
1854
|
if (!Number.isNaN(clock) && clock > maxClock) maxClock = clock;
|
|
1635
1855
|
}
|
|
1636
|
-
if (remaining.size === 0) {
|
|
1856
|
+
if (remaining.size === 0 && !settled) {
|
|
1857
|
+
settled = true;
|
|
1858
|
+
clearTimeout(timeoutTimer);
|
|
1637
1859
|
cleanup();
|
|
1638
1860
|
resolve();
|
|
1639
1861
|
}
|
|
1640
1862
|
}
|
|
1641
1863
|
})
|
|
1642
1864
|
).catch((err) => {
|
|
1865
|
+
if (settled) return;
|
|
1866
|
+
settled = true;
|
|
1867
|
+
clearTimeout(timeoutTimer);
|
|
1643
1868
|
cleanup();
|
|
1644
1869
|
reject(err);
|
|
1645
1870
|
});
|
|
@@ -1655,14 +1880,14 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1655
1880
|
);
|
|
1656
1881
|
}
|
|
1657
1882
|
}
|
|
1658
|
-
function wrapWithTimeoutWarning(logger, fn, timeoutMs,
|
|
1883
|
+
function wrapWithTimeoutWarning(logger, fn, timeoutMs, topic) {
|
|
1659
1884
|
let timer;
|
|
1660
1885
|
const promise = fn().finally(() => {
|
|
1661
1886
|
if (timer !== void 0) clearTimeout(timer);
|
|
1662
1887
|
});
|
|
1663
1888
|
timer = setTimeout(() => {
|
|
1664
1889
|
logger.warn(
|
|
1665
|
-
`Handler for topic "${
|
|
1890
|
+
`Handler for topic "${topic}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
|
|
1666
1891
|
);
|
|
1667
1892
|
}, timeoutMs);
|
|
1668
1893
|
return promise;
|
|
@@ -1680,6 +1905,15 @@ async function preparePayload(ctx, topicOrDesc, messages, compression) {
|
|
|
1680
1905
|
await ensureTopic(ctx, payload.topic);
|
|
1681
1906
|
return payload;
|
|
1682
1907
|
}
|
|
1908
|
+
async function redirectToDelayed(ctx, payload, deliverAfterMs) {
|
|
1909
|
+
const until = String(Date.now() + deliverAfterMs);
|
|
1910
|
+
for (const m of payload.messages) {
|
|
1911
|
+
m.headers[HEADER_DELAYED_UNTIL] = until;
|
|
1912
|
+
m.headers[HEADER_DELAYED_TARGET] = payload.topic;
|
|
1913
|
+
}
|
|
1914
|
+
payload.topic = `${payload.topic}.delayed`;
|
|
1915
|
+
await ensureTopic(ctx, payload.topic);
|
|
1916
|
+
}
|
|
1683
1917
|
async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
|
|
1684
1918
|
await waitIfThrottled(ctx);
|
|
1685
1919
|
const payload = await preparePayload(
|
|
@@ -1697,6 +1931,9 @@ async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
|
|
|
1697
1931
|
],
|
|
1698
1932
|
options.compression
|
|
1699
1933
|
);
|
|
1934
|
+
if (options.deliverAfterMs && options.deliverAfterMs > 0) {
|
|
1935
|
+
await redirectToDelayed(ctx, payload, options.deliverAfterMs);
|
|
1936
|
+
}
|
|
1700
1937
|
await ctx.producer.send(payload);
|
|
1701
1938
|
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1702
1939
|
}
|
|
@@ -1708,19 +1945,22 @@ async function sendBatchImpl(ctx, topicOrDesc, messages, options) {
|
|
|
1708
1945
|
messages,
|
|
1709
1946
|
options?.compression
|
|
1710
1947
|
);
|
|
1948
|
+
if (options?.deliverAfterMs && options.deliverAfterMs > 0) {
|
|
1949
|
+
await redirectToDelayed(ctx, payload, options.deliverAfterMs);
|
|
1950
|
+
}
|
|
1711
1951
|
await ctx.producer.send(payload);
|
|
1712
1952
|
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1713
1953
|
}
|
|
1714
|
-
async function sendTombstoneImpl(ctx,
|
|
1954
|
+
async function sendTombstoneImpl(ctx, topic, key, headers) {
|
|
1715
1955
|
await waitIfThrottled(ctx);
|
|
1716
1956
|
const hdrs = { ...headers };
|
|
1717
|
-
for (const inst of ctx.instrumentation) inst.beforeSend?.(
|
|
1718
|
-
await ensureTopic(ctx,
|
|
1957
|
+
for (const inst of ctx.instrumentation) inst.beforeSend?.(topic, hdrs);
|
|
1958
|
+
await ensureTopic(ctx, topic);
|
|
1719
1959
|
await ctx.producer.send({
|
|
1720
|
-
topic
|
|
1960
|
+
topic,
|
|
1721
1961
|
messages: [{ value: null, key, headers: hdrs }]
|
|
1722
1962
|
});
|
|
1723
|
-
for (const inst of ctx.instrumentation) inst.afterSend?.(
|
|
1963
|
+
for (const inst of ctx.instrumentation) inst.afterSend?.(topic);
|
|
1724
1964
|
}
|
|
1725
1965
|
async function transactionImpl(ctx, fn) {
|
|
1726
1966
|
if (!ctx.txProducerInitPromise) {
|
|
@@ -1744,6 +1984,17 @@ async function transactionImpl(ctx, fn) {
|
|
|
1744
1984
|
});
|
|
1745
1985
|
}
|
|
1746
1986
|
ctx.txProducer = await ctx.txProducerInitPromise;
|
|
1987
|
+
const prev = ctx._txChain;
|
|
1988
|
+
let release;
|
|
1989
|
+
ctx._txChain = new Promise((r) => release = r);
|
|
1990
|
+
await prev;
|
|
1991
|
+
try {
|
|
1992
|
+
await runTransaction(ctx, fn);
|
|
1993
|
+
} finally {
|
|
1994
|
+
release();
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
async function runTransaction(ctx, fn) {
|
|
1747
1998
|
const tx = await ctx.txProducer.transaction();
|
|
1748
1999
|
try {
|
|
1749
2000
|
const txCtx = {
|
|
@@ -1817,7 +2068,7 @@ async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs =
|
|
|
1817
2068
|
`Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
|
|
1818
2069
|
);
|
|
1819
2070
|
}
|
|
1820
|
-
async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
2071
|
+
async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
|
|
1821
2072
|
const {
|
|
1822
2073
|
logger,
|
|
1823
2074
|
producer,
|
|
@@ -1863,20 +2114,35 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
1863
2114
|
await sleep(remaining);
|
|
1864
2115
|
consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
|
|
1865
2116
|
}
|
|
1866
|
-
const
|
|
1867
|
-
const parsed = parseJsonMessage(raw, levelTopic, logger);
|
|
1868
|
-
if (parsed === null) {
|
|
1869
|
-
await consumer.commitOffsets([nextOffset]);
|
|
1870
|
-
return;
|
|
1871
|
-
}
|
|
2117
|
+
const rawBytes = message.value;
|
|
1872
2118
|
const currentMaxRetries = parseInt(
|
|
1873
2119
|
headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
|
|
1874
2120
|
10
|
|
1875
2121
|
);
|
|
1876
2122
|
const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
|
|
2123
|
+
const serde = serdeMap?.get(originalTopic) ?? deps.serde;
|
|
2124
|
+
let parsed;
|
|
2125
|
+
try {
|
|
2126
|
+
parsed = await serde.deserialize(rawBytes, {
|
|
2127
|
+
topic: originalTopic,
|
|
2128
|
+
headers,
|
|
2129
|
+
isKey: false
|
|
2130
|
+
});
|
|
2131
|
+
} catch (err) {
|
|
2132
|
+
logger.error(
|
|
2133
|
+
`Failed to deserialize retry message from topic ${levelTopic}:`,
|
|
2134
|
+
toError(err).stack
|
|
2135
|
+
);
|
|
2136
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2137
|
+
return;
|
|
2138
|
+
}
|
|
2139
|
+
if (parsed === null) {
|
|
2140
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2141
|
+
return;
|
|
2142
|
+
}
|
|
1877
2143
|
const validated = await validateWithSchema(
|
|
1878
2144
|
parsed,
|
|
1879
|
-
|
|
2145
|
+
rawBytes,
|
|
1880
2146
|
originalTopic,
|
|
1881
2147
|
schemaMap,
|
|
1882
2148
|
interceptors,
|
|
@@ -1911,6 +2177,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
1911
2177
|
await consumer.commitOffsets([nextOffset]);
|
|
1912
2178
|
return;
|
|
1913
2179
|
}
|
|
2180
|
+
deps.onFailure?.(envelope);
|
|
1914
2181
|
const exhausted = level >= currentMaxRetries;
|
|
1915
2182
|
const reportedError = exhausted && currentMaxRetries > 1 ? new KafkaRetryExhaustedError(
|
|
1916
2183
|
originalTopic,
|
|
@@ -1929,7 +2196,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
1929
2196
|
const delay = Math.floor(Math.random() * cap);
|
|
1930
2197
|
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
1931
2198
|
originalTopic,
|
|
1932
|
-
[
|
|
2199
|
+
[rawBytes],
|
|
1933
2200
|
nextLevel,
|
|
1934
2201
|
currentMaxRetries,
|
|
1935
2202
|
delay,
|
|
@@ -1971,7 +2238,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
1971
2238
|
} else if (dlq) {
|
|
1972
2239
|
const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
|
|
1973
2240
|
originalTopic,
|
|
1974
|
-
|
|
2241
|
+
rawBytes,
|
|
1975
2242
|
{
|
|
1976
2243
|
error,
|
|
1977
2244
|
// +1 to account for the main consumer's initial attempt before routing.
|
|
@@ -2033,7 +2300,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2033
2300
|
`Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
|
|
2034
2301
|
);
|
|
2035
2302
|
}
|
|
2036
|
-
async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
2303
|
+
async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
|
|
2037
2304
|
const levelGroupIds = new Array(retry.maxRetries);
|
|
2038
2305
|
await Promise.all(
|
|
2039
2306
|
Array.from({ length: retry.maxRetries }, async (_, i) => {
|
|
@@ -2051,7 +2318,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
2051
2318
|
interceptors,
|
|
2052
2319
|
schemaMap,
|
|
2053
2320
|
deps,
|
|
2054
|
-
assignmentTimeoutMs
|
|
2321
|
+
assignmentTimeoutMs,
|
|
2322
|
+
serdeMap
|
|
2055
2323
|
);
|
|
2056
2324
|
levelGroupIds[i] = levelGroupId;
|
|
2057
2325
|
})
|
|
@@ -2126,7 +2394,8 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2126
2394
|
options.autoCommit ?? true,
|
|
2127
2395
|
ctx.consumerOpsDeps,
|
|
2128
2396
|
options.partitionAssigner,
|
|
2129
|
-
resolveReady
|
|
2397
|
+
resolveReady,
|
|
2398
|
+
options.groupInstanceId
|
|
2130
2399
|
);
|
|
2131
2400
|
const schemaMap = buildSchemaMap(
|
|
2132
2401
|
stringTopics,
|
|
@@ -2134,10 +2403,14 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2134
2403
|
optionSchemas,
|
|
2135
2404
|
ctx.logger
|
|
2136
2405
|
);
|
|
2406
|
+
const serdeMap = buildSerdeMap(stringTopics);
|
|
2137
2407
|
const topicNames = stringTopics.map((t) => resolveTopicName(t));
|
|
2138
2408
|
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2139
2409
|
await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
|
|
2140
2410
|
await consumer.connect();
|
|
2411
|
+
if (dlq || options.retryTopics || options.deduplication) {
|
|
2412
|
+
await ctx.producer.connect();
|
|
2413
|
+
}
|
|
2141
2414
|
await subscribeWithRetry(
|
|
2142
2415
|
consumer,
|
|
2143
2416
|
subscribeTopics,
|
|
@@ -2148,19 +2421,19 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2148
2421
|
ctx.logger.log(
|
|
2149
2422
|
`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
|
|
2150
2423
|
);
|
|
2151
|
-
return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
|
|
2424
|
+
return { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
|
|
2152
2425
|
}
|
|
2153
2426
|
function resolveDeduplicationContext(ctx, groupId, options) {
|
|
2154
2427
|
if (!options) return void 0;
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
return { options, state: ctx.dedupStates.get(groupId) };
|
|
2428
|
+
const store = options.store ?? new InMemoryDedupStore(ctx.dedupStates);
|
|
2429
|
+
return { options, store, groupId };
|
|
2158
2430
|
}
|
|
2159
2431
|
function messageDepsFor(ctx, gid, options) {
|
|
2160
2432
|
const notifyRetry = ctx.metrics.notifyRetry.bind(ctx.metrics);
|
|
2161
2433
|
return {
|
|
2162
2434
|
logger: ctx.logger,
|
|
2163
2435
|
producer: ctx.producer,
|
|
2436
|
+
serde: ctx.serde,
|
|
2164
2437
|
instrumentation: ctx.instrumentation,
|
|
2165
2438
|
onMessageLost: options?.onMessageLost ?? ctx.onMessageLost,
|
|
2166
2439
|
onTtlExpired: ctx.onTtlExpired,
|
|
@@ -2168,15 +2441,17 @@ function messageDepsFor(ctx, gid, options) {
|
|
|
2168
2441
|
notifyRetry(envelope, attempt, max);
|
|
2169
2442
|
return options.onRetry(envelope, attempt, max);
|
|
2170
2443
|
} : notifyRetry,
|
|
2171
|
-
onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason
|
|
2444
|
+
onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason),
|
|
2172
2445
|
onDuplicate: ctx.metrics.notifyDuplicate.bind(ctx.metrics),
|
|
2173
|
-
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid)
|
|
2446
|
+
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
|
|
2447
|
+
onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid)
|
|
2174
2448
|
};
|
|
2175
2449
|
}
|
|
2176
2450
|
function buildRetryTopicDeps(ctx) {
|
|
2177
2451
|
return {
|
|
2178
2452
|
logger: ctx.logger,
|
|
2179
2453
|
producer: ctx.producer,
|
|
2454
|
+
serde: ctx.serde,
|
|
2180
2455
|
instrumentation: ctx.instrumentation,
|
|
2181
2456
|
onMessageLost: ctx.onMessageLost,
|
|
2182
2457
|
onRetry: ctx.metrics.notifyRetry.bind(ctx.metrics),
|
|
@@ -2194,7 +2469,7 @@ async function makeEosMainContext(ctx, gid, consumer, options) {
|
|
|
2194
2469
|
return { txProducer, consumer };
|
|
2195
2470
|
}
|
|
2196
2471
|
async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
2197
|
-
const { retry, dlq, interceptors, schemaMap, assignmentTimeoutMs } = opts;
|
|
2472
|
+
const { retry, dlq, interceptors, schemaMap, serdeMap, assignmentTimeoutMs } = opts;
|
|
2198
2473
|
if (!ctx.autoCreateTopicsEnabled) {
|
|
2199
2474
|
await ctx.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
2200
2475
|
}
|
|
@@ -2209,11 +2484,17 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
|
2209
2484
|
schemaMap,
|
|
2210
2485
|
{
|
|
2211
2486
|
...ctx.retryTopicDeps,
|
|
2487
|
+
// Bind circuit breaker events to the MAIN consumer group so failures and
|
|
2488
|
+
// successes inside the retry chain drive the same breaker as the main
|
|
2489
|
+
// consumer (the retry chain has no breaker config of its own).
|
|
2490
|
+
onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid),
|
|
2491
|
+
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
|
|
2212
2492
|
onLevelStarted: (levelGroupId) => {
|
|
2213
2493
|
ctx.companionGroupIds.get(gid).push(levelGroupId);
|
|
2214
2494
|
}
|
|
2215
2495
|
},
|
|
2216
|
-
assignmentTimeoutMs
|
|
2496
|
+
assignmentTimeoutMs,
|
|
2497
|
+
serdeMap
|
|
2217
2498
|
);
|
|
2218
2499
|
}
|
|
2219
2500
|
|
|
@@ -2224,7 +2505,15 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
2224
2505
|
const incomingClock = Number(clockRaw);
|
|
2225
2506
|
if (Number.isNaN(incomingClock)) return false;
|
|
2226
2507
|
const stateKey = `${envelope.topic}:${envelope.partition}`;
|
|
2227
|
-
|
|
2508
|
+
let lastProcessedClock;
|
|
2509
|
+
try {
|
|
2510
|
+
lastProcessedClock = await dedup.store.getLastClock(dedup.groupId, stateKey) ?? -1;
|
|
2511
|
+
} catch (err) {
|
|
2512
|
+
deps.logger.error(
|
|
2513
|
+
`Dedup store getLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 treating message as not a duplicate (fail-open): ${err.message}`
|
|
2514
|
+
);
|
|
2515
|
+
return false;
|
|
2516
|
+
}
|
|
2228
2517
|
if (incomingClock <= lastProcessedClock) {
|
|
2229
2518
|
const meta = {
|
|
2230
2519
|
incomingClock,
|
|
@@ -2254,34 +2543,52 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
2254
2543
|
}
|
|
2255
2544
|
return true;
|
|
2256
2545
|
}
|
|
2257
|
-
|
|
2546
|
+
try {
|
|
2547
|
+
await dedup.store.setLastClock(dedup.groupId, stateKey, incomingClock);
|
|
2548
|
+
} catch (err) {
|
|
2549
|
+
deps.logger.error(
|
|
2550
|
+
`Dedup store setLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 processing message anyway (fail-open): ${err.message}`
|
|
2551
|
+
);
|
|
2552
|
+
}
|
|
2258
2553
|
return false;
|
|
2259
2554
|
}
|
|
2260
|
-
async function parseSingleMessage(message,
|
|
2555
|
+
async function parseSingleMessage(message, topic, partition, schemaMap, interceptors, dlq, deps, serdeMap) {
|
|
2261
2556
|
if (!message.value) {
|
|
2262
|
-
deps.logger.warn(`Received empty message from topic ${
|
|
2557
|
+
deps.logger.warn(`Received empty message from topic ${topic}`);
|
|
2263
2558
|
return null;
|
|
2264
2559
|
}
|
|
2265
|
-
const
|
|
2266
|
-
const parsed = parseJsonMessage(raw, topic2, deps.logger);
|
|
2267
|
-
if (parsed === null) return null;
|
|
2560
|
+
const bytes = message.value;
|
|
2268
2561
|
const headers = decodeHeaders(message.headers);
|
|
2562
|
+
const serde = serdeMap?.get(topic) ?? deps.serde;
|
|
2563
|
+
let parsed;
|
|
2564
|
+
try {
|
|
2565
|
+
parsed = await serde.deserialize(bytes, { topic, headers, isKey: false });
|
|
2566
|
+
} catch (error) {
|
|
2567
|
+
deps.logger.error(
|
|
2568
|
+
`Failed to deserialize message from topic ${topic}:`,
|
|
2569
|
+
toError(error).stack
|
|
2570
|
+
);
|
|
2571
|
+
return null;
|
|
2572
|
+
}
|
|
2573
|
+
if (parsed === null) return null;
|
|
2269
2574
|
const validated = await validateWithSchema(
|
|
2270
2575
|
parsed,
|
|
2271
|
-
|
|
2272
|
-
|
|
2576
|
+
// Forward the ORIGINAL bytes to DLQ on validation failure (binary-safe).
|
|
2577
|
+
bytes,
|
|
2578
|
+
topic,
|
|
2273
2579
|
schemaMap,
|
|
2274
2580
|
interceptors,
|
|
2275
2581
|
dlq,
|
|
2276
2582
|
{ ...deps, originalHeaders: headers }
|
|
2277
2583
|
);
|
|
2278
2584
|
if (validated === null) return null;
|
|
2279
|
-
return extractEnvelope(validated, headers,
|
|
2585
|
+
return extractEnvelope(validated, headers, topic, partition, message.offset);
|
|
2280
2586
|
}
|
|
2281
2587
|
async function handleEachMessage(payload, opts, deps) {
|
|
2282
|
-
const { topic
|
|
2588
|
+
const { topic, partition, message } = payload;
|
|
2283
2589
|
const {
|
|
2284
2590
|
schemaMap,
|
|
2591
|
+
serdeMap,
|
|
2285
2592
|
handleMessage,
|
|
2286
2593
|
interceptors,
|
|
2287
2594
|
dlq,
|
|
@@ -2290,16 +2597,17 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2290
2597
|
timeoutMs,
|
|
2291
2598
|
wrapWithTimeout
|
|
2292
2599
|
} = opts;
|
|
2600
|
+
const rawBytes = message.value;
|
|
2293
2601
|
const eos = opts.eosMainContext;
|
|
2294
2602
|
const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
|
|
2295
2603
|
const commitOffset = eos ? async () => {
|
|
2296
2604
|
await eos.consumer.commitOffsets([
|
|
2297
|
-
{ topic
|
|
2605
|
+
{ topic, partition, offset: nextOffsetStr }
|
|
2298
2606
|
]);
|
|
2299
2607
|
} : void 0;
|
|
2300
2608
|
const eosRouteToRetry = eos && retry ? async (rawMsgs, envelopes, delay) => {
|
|
2301
2609
|
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
2302
|
-
|
|
2610
|
+
topic,
|
|
2303
2611
|
rawMsgs,
|
|
2304
2612
|
1,
|
|
2305
2613
|
retry.maxRetries,
|
|
@@ -2313,7 +2621,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2313
2621
|
consumer: eos.consumer,
|
|
2314
2622
|
topics: [
|
|
2315
2623
|
{
|
|
2316
|
-
topic
|
|
2624
|
+
topic,
|
|
2317
2625
|
partitions: [{ partition, offset: nextOffsetStr }]
|
|
2318
2626
|
}
|
|
2319
2627
|
]
|
|
@@ -2329,12 +2637,13 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2329
2637
|
} : void 0;
|
|
2330
2638
|
const envelope = await parseSingleMessage(
|
|
2331
2639
|
message,
|
|
2332
|
-
|
|
2640
|
+
topic,
|
|
2333
2641
|
partition,
|
|
2334
2642
|
schemaMap,
|
|
2335
2643
|
interceptors,
|
|
2336
2644
|
dlq,
|
|
2337
|
-
deps
|
|
2645
|
+
deps,
|
|
2646
|
+
serdeMap
|
|
2338
2647
|
);
|
|
2339
2648
|
if (envelope === null) {
|
|
2340
2649
|
await commitOffset?.();
|
|
@@ -2343,7 +2652,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2343
2652
|
if (opts.deduplication) {
|
|
2344
2653
|
const isDuplicate = await applyDeduplication(
|
|
2345
2654
|
envelope,
|
|
2346
|
-
|
|
2655
|
+
rawBytes,
|
|
2347
2656
|
opts.deduplication,
|
|
2348
2657
|
dlq,
|
|
2349
2658
|
deps
|
|
@@ -2357,10 +2666,10 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2357
2666
|
const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
|
|
2358
2667
|
if (ageMs > opts.messageTtlMs) {
|
|
2359
2668
|
deps.logger.warn(
|
|
2360
|
-
`[KafkaClient] TTL expired on ${
|
|
2669
|
+
`[KafkaClient] TTL expired on ${topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2361
2670
|
);
|
|
2362
2671
|
if (dlq) {
|
|
2363
|
-
await sendToDlq(
|
|
2672
|
+
await sendToDlq(topic, rawBytes, deps, {
|
|
2364
2673
|
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2365
2674
|
attempt: 0,
|
|
2366
2675
|
originalHeaders: envelope.headers
|
|
@@ -2369,7 +2678,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2369
2678
|
} else {
|
|
2370
2679
|
const ttlHandler = opts.onTtlExpired ?? deps.onTtlExpired;
|
|
2371
2680
|
await ttlHandler?.({
|
|
2372
|
-
topic
|
|
2681
|
+
topic,
|
|
2373
2682
|
ageMs,
|
|
2374
2683
|
messageTtlMs: opts.messageTtlMs,
|
|
2375
2684
|
headers: envelope.headers
|
|
@@ -2388,11 +2697,11 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2388
2697
|
},
|
|
2389
2698
|
() => handleMessage(envelope)
|
|
2390
2699
|
);
|
|
2391
|
-
return timeoutMs ? wrapWithTimeout(fn, timeoutMs,
|
|
2700
|
+
return timeoutMs ? wrapWithTimeout(fn, timeoutMs, topic) : fn();
|
|
2392
2701
|
},
|
|
2393
2702
|
{
|
|
2394
2703
|
envelope,
|
|
2395
|
-
rawMessages: [
|
|
2704
|
+
rawMessages: [rawBytes],
|
|
2396
2705
|
interceptors,
|
|
2397
2706
|
dlq,
|
|
2398
2707
|
retry,
|
|
@@ -2405,6 +2714,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2405
2714
|
const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
|
|
2406
2715
|
const {
|
|
2407
2716
|
schemaMap,
|
|
2717
|
+
serdeMap,
|
|
2408
2718
|
handleBatch,
|
|
2409
2719
|
interceptors,
|
|
2410
2720
|
dlq,
|
|
@@ -2460,6 +2770,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2460
2770
|
const envelopes = [];
|
|
2461
2771
|
const rawMessages = [];
|
|
2462
2772
|
for (const message of batch.messages) {
|
|
2773
|
+
const rawBytes = message.value;
|
|
2463
2774
|
const envelope = await parseSingleMessage(
|
|
2464
2775
|
message,
|
|
2465
2776
|
batch.topic,
|
|
@@ -2467,14 +2778,14 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2467
2778
|
schemaMap,
|
|
2468
2779
|
interceptors,
|
|
2469
2780
|
dlq,
|
|
2470
|
-
deps
|
|
2781
|
+
deps,
|
|
2782
|
+
serdeMap
|
|
2471
2783
|
);
|
|
2472
2784
|
if (envelope === null) continue;
|
|
2473
2785
|
if (opts.deduplication) {
|
|
2474
|
-
const raw = message.value.toString();
|
|
2475
2786
|
const isDuplicate = await applyDeduplication(
|
|
2476
2787
|
envelope,
|
|
2477
|
-
|
|
2788
|
+
rawBytes,
|
|
2478
2789
|
opts.deduplication,
|
|
2479
2790
|
dlq,
|
|
2480
2791
|
deps
|
|
@@ -2488,7 +2799,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2488
2799
|
`[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2489
2800
|
);
|
|
2490
2801
|
if (dlq) {
|
|
2491
|
-
await sendToDlq(batch.topic,
|
|
2802
|
+
await sendToDlq(batch.topic, rawBytes, deps, {
|
|
2492
2803
|
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2493
2804
|
attempt: 0,
|
|
2494
2805
|
originalHeaders: envelope.headers
|
|
@@ -2507,7 +2818,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2507
2818
|
}
|
|
2508
2819
|
}
|
|
2509
2820
|
envelopes.push(envelope);
|
|
2510
|
-
rawMessages.push(
|
|
2821
|
+
rawMessages.push(rawBytes);
|
|
2511
2822
|
}
|
|
2512
2823
|
if (envelopes.length === 0) {
|
|
2513
2824
|
await commitBatchOffset?.();
|
|
@@ -2632,7 +2943,7 @@ function pauseConsumerImpl(ctx, groupId, assignments) {
|
|
|
2632
2943
|
}
|
|
2633
2944
|
consumer.pause(
|
|
2634
2945
|
assignments.flatMap(
|
|
2635
|
-
({ topic
|
|
2946
|
+
({ topic, partitions }) => partitions.map((p) => ({ topic, partitions: [p] }))
|
|
2636
2947
|
)
|
|
2637
2948
|
);
|
|
2638
2949
|
}
|
|
@@ -2645,32 +2956,32 @@ function resumeConsumerImpl(ctx, groupId, assignments) {
|
|
|
2645
2956
|
}
|
|
2646
2957
|
consumer.resume(
|
|
2647
2958
|
assignments.flatMap(
|
|
2648
|
-
({ topic
|
|
2959
|
+
({ topic, partitions }) => partitions.map((p) => ({ topic, partitions: [p] }))
|
|
2649
2960
|
)
|
|
2650
2961
|
);
|
|
2651
2962
|
}
|
|
2652
|
-
function pauseTopicAllPartitions(ctx, gid,
|
|
2963
|
+
function pauseTopicAllPartitions(ctx, gid, topic) {
|
|
2653
2964
|
const consumer = ctx.consumers.get(gid);
|
|
2654
2965
|
if (!consumer) return;
|
|
2655
2966
|
const assignment = consumer.assignment();
|
|
2656
|
-
const partitions = assignment.filter((a) => a.topic ===
|
|
2967
|
+
const partitions = assignment.filter((a) => a.topic === topic).map((a) => a.partition);
|
|
2657
2968
|
if (partitions.length > 0)
|
|
2658
|
-
consumer.pause(partitions.map((p) => ({ topic
|
|
2969
|
+
consumer.pause(partitions.map((p) => ({ topic, partitions: [p] })));
|
|
2659
2970
|
}
|
|
2660
|
-
function resumeTopicAllPartitions(ctx, gid,
|
|
2971
|
+
function resumeTopicAllPartitions(ctx, gid, topic) {
|
|
2661
2972
|
const consumer = ctx.consumers.get(gid);
|
|
2662
2973
|
if (!consumer) return;
|
|
2663
2974
|
const assignment = consumer.assignment();
|
|
2664
|
-
const partitions = assignment.filter((a) => a.topic ===
|
|
2975
|
+
const partitions = assignment.filter((a) => a.topic === topic).map((a) => a.partition);
|
|
2665
2976
|
if (partitions.length > 0)
|
|
2666
|
-
consumer.resume(partitions.map((p) => ({ topic
|
|
2977
|
+
consumer.resume(partitions.map((p) => ({ topic, partitions: [p] })));
|
|
2667
2978
|
}
|
|
2668
2979
|
|
|
2669
2980
|
// src/client/kafka.client/consumer/start.ts
|
|
2670
2981
|
async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
2671
2982
|
validateTopicConsumerOpts(topics, options);
|
|
2672
2983
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
2673
|
-
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
|
|
2984
|
+
const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
|
|
2674
2985
|
if (options.circuitBreaker)
|
|
2675
2986
|
ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
2676
2987
|
const deps = messageDepsFor(ctx, gid, options);
|
|
@@ -2681,13 +2992,14 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
|
2681
2992
|
payload,
|
|
2682
2993
|
{
|
|
2683
2994
|
schemaMap,
|
|
2995
|
+
serdeMap,
|
|
2684
2996
|
handleMessage,
|
|
2685
2997
|
interceptors,
|
|
2686
2998
|
dlq,
|
|
2687
2999
|
retry,
|
|
2688
3000
|
retryTopics: options.retryTopics,
|
|
2689
3001
|
timeoutMs: options.handlerTimeoutMs,
|
|
2690
|
-
wrapWithTimeout: (fn, ms,
|
|
3002
|
+
wrapWithTimeout: (fn, ms, topic) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic),
|
|
2691
3003
|
deduplication: resolveDeduplicationContext(
|
|
2692
3004
|
ctx,
|
|
2693
3005
|
gid,
|
|
@@ -2708,6 +3020,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
|
2708
3020
|
dlq,
|
|
2709
3021
|
interceptors,
|
|
2710
3022
|
schemaMap,
|
|
3023
|
+
serdeMap,
|
|
2711
3024
|
assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
|
|
2712
3025
|
});
|
|
2713
3026
|
}
|
|
@@ -2721,7 +3034,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
2721
3034
|
);
|
|
2722
3035
|
}
|
|
2723
3036
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
2724
|
-
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
|
|
3037
|
+
const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
|
|
2725
3038
|
if (options.circuitBreaker)
|
|
2726
3039
|
ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
2727
3040
|
const deps = messageDepsFor(ctx, gid, options);
|
|
@@ -2732,13 +3045,14 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
2732
3045
|
payload,
|
|
2733
3046
|
{
|
|
2734
3047
|
schemaMap,
|
|
3048
|
+
serdeMap,
|
|
2735
3049
|
handleBatch,
|
|
2736
3050
|
interceptors,
|
|
2737
3051
|
dlq,
|
|
2738
3052
|
retry,
|
|
2739
3053
|
retryTopics: options.retryTopics,
|
|
2740
3054
|
timeoutMs: options.handlerTimeoutMs,
|
|
2741
|
-
wrapWithTimeout: (fn, ms,
|
|
3055
|
+
wrapWithTimeout: (fn, ms, topic) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic),
|
|
2742
3056
|
deduplication: resolveDeduplicationContext(
|
|
2743
3057
|
ctx,
|
|
2744
3058
|
gid,
|
|
@@ -2769,6 +3083,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
2769
3083
|
dlq,
|
|
2770
3084
|
interceptors,
|
|
2771
3085
|
schemaMap,
|
|
3086
|
+
serdeMap,
|
|
2772
3087
|
assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
|
|
2773
3088
|
});
|
|
2774
3089
|
}
|
|
@@ -2781,7 +3096,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
2781
3096
|
);
|
|
2782
3097
|
}
|
|
2783
3098
|
const setupOptions = { ...options, autoCommit: false };
|
|
2784
|
-
const { consumer, schemaMap, gid, readyPromise } = await setupConsumer(
|
|
3099
|
+
const { consumer, schemaMap, serdeMap, gid, readyPromise } = await setupConsumer(
|
|
2785
3100
|
ctx,
|
|
2786
3101
|
topics,
|
|
2787
3102
|
"eachMessage",
|
|
@@ -2790,20 +3105,21 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
2790
3105
|
const txProducer = await createRetryTxProducer(ctx, `${gid}-txc`);
|
|
2791
3106
|
const deps = messageDepsFor(ctx, gid);
|
|
2792
3107
|
await consumer.run({
|
|
2793
|
-
eachMessage: ({ topic
|
|
3108
|
+
eachMessage: ({ topic, partition, message }) => ctx.inFlight.track(async () => {
|
|
2794
3109
|
const envelope = await parseSingleMessage(
|
|
2795
3110
|
message,
|
|
2796
|
-
|
|
3111
|
+
topic,
|
|
2797
3112
|
partition,
|
|
2798
3113
|
schemaMap,
|
|
2799
3114
|
options.interceptors ?? [],
|
|
2800
3115
|
false,
|
|
2801
|
-
deps
|
|
3116
|
+
deps,
|
|
3117
|
+
serdeMap
|
|
2802
3118
|
);
|
|
2803
3119
|
const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
|
|
2804
3120
|
if (envelope === null) {
|
|
2805
3121
|
await consumer.commitOffsets([
|
|
2806
|
-
{ topic
|
|
3122
|
+
{ topic, partition, offset: nextOffset }
|
|
2807
3123
|
]);
|
|
2808
3124
|
return;
|
|
2809
3125
|
}
|
|
@@ -2843,7 +3159,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
2843
3159
|
await tx.sendOffsets({
|
|
2844
3160
|
consumer,
|
|
2845
3161
|
topics: [
|
|
2846
|
-
{ topic
|
|
3162
|
+
{ topic, partitions: [{ partition, offset: nextOffset }] }
|
|
2847
3163
|
]
|
|
2848
3164
|
});
|
|
2849
3165
|
await tx.commit();
|
|
@@ -2854,7 +3170,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
2854
3170
|
} catch {
|
|
2855
3171
|
}
|
|
2856
3172
|
ctx.logger.warn(
|
|
2857
|
-
`startTransactionalConsumer: handler failed on ${
|
|
3173
|
+
`startTransactionalConsumer: handler failed on ${topic}[${partition}]@${message.offset} \u2014 tx aborted, message will be redelivered (${toError(err).message})`
|
|
2858
3174
|
);
|
|
2859
3175
|
throw err;
|
|
2860
3176
|
}
|
|
@@ -2867,8 +3183,8 @@ function stopConsumerByGid(ctx, gid) {
|
|
|
2867
3183
|
return stopConsumerImpl(ctx, gid);
|
|
2868
3184
|
}
|
|
2869
3185
|
|
|
2870
|
-
// src/client/kafka.client/consumer/window.ts
|
|
2871
|
-
async function startWindowConsumerImpl(ctx,
|
|
3186
|
+
// src/client/kafka.client/consumer/features/window.ts
|
|
3187
|
+
async function startWindowConsumerImpl(ctx, topic, handler, options) {
|
|
2872
3188
|
const { maxMessages, maxMs, ...consumerOptions } = options;
|
|
2873
3189
|
if (maxMessages <= 0)
|
|
2874
3190
|
throw new Error("startWindowConsumer: maxMessages must be > 0");
|
|
@@ -2881,6 +3197,7 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2881
3197
|
const buffer = [];
|
|
2882
3198
|
let flushTimer = null;
|
|
2883
3199
|
let windowStart = 0;
|
|
3200
|
+
const onLost = consumerOptions.onMessageLost ?? ctx.onMessageLost;
|
|
2884
3201
|
const flush = async (trigger) => {
|
|
2885
3202
|
if (flushTimer !== null) {
|
|
2886
3203
|
clearTimeout(flushTimer);
|
|
@@ -2888,22 +3205,37 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2888
3205
|
}
|
|
2889
3206
|
if (buffer.length === 0) return;
|
|
2890
3207
|
const envelopes = buffer.splice(0);
|
|
2891
|
-
|
|
3208
|
+
try {
|
|
3209
|
+
await handler(envelopes, { trigger, windowStart, windowEnd: Date.now() });
|
|
3210
|
+
} catch (err) {
|
|
3211
|
+
const error = toError(err);
|
|
3212
|
+
ctx.logger.error(
|
|
3213
|
+
`startWindowConsumer: ${trigger}-triggered flush failed \u2014 window of ${envelopes.length} message(s) lost:`,
|
|
3214
|
+
error.stack
|
|
3215
|
+
);
|
|
3216
|
+
for (const envelope of envelopes) {
|
|
3217
|
+
await Promise.resolve(
|
|
3218
|
+
onLost?.({
|
|
3219
|
+
topic: envelope.topic,
|
|
3220
|
+
error,
|
|
3221
|
+
attempt: 0,
|
|
3222
|
+
headers: envelope.headers
|
|
3223
|
+
})
|
|
3224
|
+
).catch(() => {
|
|
3225
|
+
});
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
2892
3228
|
};
|
|
2893
3229
|
const scheduleFlush = () => {
|
|
2894
3230
|
if (flushTimer !== null) return;
|
|
2895
3231
|
flushTimer = setTimeout(() => {
|
|
2896
3232
|
flushTimer = null;
|
|
2897
|
-
flush("time")
|
|
2898
|
-
ctx.logger.warn(
|
|
2899
|
-
`startWindowConsumer: time-triggered flush error \u2014 ${toError(err).message}`
|
|
2900
|
-
);
|
|
2901
|
-
});
|
|
3233
|
+
void flush("time");
|
|
2902
3234
|
}, maxMs);
|
|
2903
3235
|
};
|
|
2904
3236
|
const handle = await startConsumerImpl(
|
|
2905
3237
|
ctx,
|
|
2906
|
-
[
|
|
3238
|
+
[topic],
|
|
2907
3239
|
async (envelope) => {
|
|
2908
3240
|
if (buffer.length === 0) windowStart = Date.now();
|
|
2909
3241
|
buffer.push(envelope);
|
|
@@ -2914,40 +3246,13 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2914
3246
|
);
|
|
2915
3247
|
const originalStop = handle.stop.bind(handle);
|
|
2916
3248
|
handle.stop = async () => {
|
|
2917
|
-
|
|
2918
|
-
clearTimeout(flushTimer);
|
|
2919
|
-
flushTimer = null;
|
|
2920
|
-
}
|
|
2921
|
-
if (buffer.length > 0) {
|
|
2922
|
-
const envelopes = buffer.splice(0);
|
|
2923
|
-
await handler(envelopes, {
|
|
2924
|
-
trigger: "time",
|
|
2925
|
-
windowStart,
|
|
2926
|
-
windowEnd: Date.now()
|
|
2927
|
-
}).catch(async (err) => {
|
|
2928
|
-
const error = toError(err);
|
|
2929
|
-
ctx.logger.warn(
|
|
2930
|
-
`startWindowConsumer: shutdown flush error \u2014 ${error.message}`
|
|
2931
|
-
);
|
|
2932
|
-
for (const envelope of envelopes) {
|
|
2933
|
-
await Promise.resolve(
|
|
2934
|
-
ctx.onMessageLost?.({
|
|
2935
|
-
topic: envelope.topic,
|
|
2936
|
-
error,
|
|
2937
|
-
attempt: 0,
|
|
2938
|
-
headers: envelope.headers
|
|
2939
|
-
})
|
|
2940
|
-
).catch(() => {
|
|
2941
|
-
});
|
|
2942
|
-
}
|
|
2943
|
-
});
|
|
2944
|
-
}
|
|
3249
|
+
await flush("time");
|
|
2945
3250
|
return originalStop();
|
|
2946
3251
|
};
|
|
2947
3252
|
return handle;
|
|
2948
3253
|
}
|
|
2949
3254
|
|
|
2950
|
-
// src/client/kafka.client/consumer/routed.ts
|
|
3255
|
+
// src/client/kafka.client/consumer/features/routed.ts
|
|
2951
3256
|
async function startRoutedConsumerImpl(ctx, topics, routing, options) {
|
|
2952
3257
|
const { header, routes, fallback } = routing;
|
|
2953
3258
|
const handleMessage = async (envelope) => {
|
|
@@ -2962,15 +3267,128 @@ async function startRoutedConsumerImpl(ctx, topics, routing, options) {
|
|
|
2962
3267
|
return startConsumerImpl(ctx, topics, handleMessage, options);
|
|
2963
3268
|
}
|
|
2964
3269
|
|
|
2965
|
-
// src/client/kafka.client/consumer/
|
|
2966
|
-
|
|
3270
|
+
// src/client/kafka.client/consumer/features/delayed.ts
|
|
3271
|
+
function delayedTopicName(topic) {
|
|
3272
|
+
return `${topic}.delayed`;
|
|
3273
|
+
}
|
|
3274
|
+
async function startDelayedRelayImpl(ctx, topics, options) {
|
|
3275
|
+
if (topics.length === 0) {
|
|
3276
|
+
throw new Error("startDelayedRelay: at least one topic is required");
|
|
3277
|
+
}
|
|
3278
|
+
const gid = options?.groupId ?? `${ctx.defaultGroupId}-delayed-relay`;
|
|
3279
|
+
if (ctx.runningConsumers.has(gid)) {
|
|
3280
|
+
throw new Error(
|
|
3281
|
+
`startDelayedRelay("${gid}") called twice \u2014 this group is already consuming. Call stopConsumer("${gid}") first or pass a different groupId.`
|
|
3282
|
+
);
|
|
3283
|
+
}
|
|
3284
|
+
const delayedTopics = topics.map(delayedTopicName);
|
|
3285
|
+
for (const t of delayedTopics) await ensureTopic(ctx, t);
|
|
3286
|
+
const txProducer = await createRetryTxProducer(ctx, `${gid}-tx`);
|
|
3287
|
+
let resolveReady;
|
|
3288
|
+
const readyPromise = new Promise((resolve) => {
|
|
3289
|
+
resolveReady = resolve;
|
|
3290
|
+
});
|
|
3291
|
+
const consumer = getOrCreateConsumer(
|
|
3292
|
+
gid,
|
|
3293
|
+
false,
|
|
3294
|
+
false,
|
|
3295
|
+
ctx.consumerOpsDeps,
|
|
3296
|
+
void 0,
|
|
3297
|
+
resolveReady
|
|
3298
|
+
);
|
|
3299
|
+
await consumer.connect();
|
|
3300
|
+
await subscribeWithRetry(consumer, delayedTopics, ctx.logger);
|
|
3301
|
+
await consumer.run({
|
|
3302
|
+
eachMessage: async ({ topic: stagingTopic, partition, message }) => {
|
|
3303
|
+
const nextOffset = {
|
|
3304
|
+
topic: stagingTopic,
|
|
3305
|
+
partition,
|
|
3306
|
+
offset: (parseInt(message.offset, 10) + 1).toString()
|
|
3307
|
+
};
|
|
3308
|
+
if (!message.value) {
|
|
3309
|
+
await consumer.commitOffsets([nextOffset]);
|
|
3310
|
+
return;
|
|
3311
|
+
}
|
|
3312
|
+
const headers = decodeHeaders(message.headers);
|
|
3313
|
+
const target = headers[HEADER_DELAYED_TARGET] ?? stagingTopic.replace(/\.delayed$/, "");
|
|
3314
|
+
const until = parseInt(
|
|
3315
|
+
headers[HEADER_DELAYED_UNTIL] ?? "0",
|
|
3316
|
+
10
|
|
3317
|
+
);
|
|
3318
|
+
const remaining = until - Date.now();
|
|
3319
|
+
if (remaining > 0) {
|
|
3320
|
+
consumer.pause([{ topic: stagingTopic, partitions: [partition] }]);
|
|
3321
|
+
await sleep(remaining);
|
|
3322
|
+
consumer.resume([{ topic: stagingTopic, partitions: [partition] }]);
|
|
3323
|
+
}
|
|
3324
|
+
const forwardHeaders = Object.fromEntries(
|
|
3325
|
+
Object.entries(headers).filter(
|
|
3326
|
+
([k]) => k !== HEADER_DELAYED_UNTIL && k !== HEADER_DELAYED_TARGET
|
|
3327
|
+
)
|
|
3328
|
+
);
|
|
3329
|
+
const tx = await txProducer.transaction();
|
|
3330
|
+
try {
|
|
3331
|
+
await tx.send({
|
|
3332
|
+
topic: target,
|
|
3333
|
+
messages: [
|
|
3334
|
+
{
|
|
3335
|
+
// Forward the ORIGINAL wire bytes unchanged — no re-serialization,
|
|
3336
|
+
// so binary payloads (Avro/Protobuf) are relayed losslessly.
|
|
3337
|
+
value: message.value,
|
|
3338
|
+
key: message.key ? message.key.toString() : null,
|
|
3339
|
+
headers: forwardHeaders
|
|
3340
|
+
}
|
|
3341
|
+
]
|
|
3342
|
+
});
|
|
3343
|
+
await tx.sendOffsets({
|
|
3344
|
+
consumer,
|
|
3345
|
+
topics: [
|
|
3346
|
+
{
|
|
3347
|
+
topic: nextOffset.topic,
|
|
3348
|
+
partitions: [
|
|
3349
|
+
{ partition: nextOffset.partition, offset: nextOffset.offset }
|
|
3350
|
+
]
|
|
3351
|
+
}
|
|
3352
|
+
]
|
|
3353
|
+
});
|
|
3354
|
+
await tx.commit();
|
|
3355
|
+
ctx.logger.debug?.(
|
|
3356
|
+
`Delayed message relayed to "${target}" (deadline ${new Date(until).toISOString()})`
|
|
3357
|
+
);
|
|
3358
|
+
} catch (txErr) {
|
|
3359
|
+
try {
|
|
3360
|
+
await tx.abort();
|
|
3361
|
+
} catch {
|
|
3362
|
+
}
|
|
3363
|
+
ctx.logger.error(
|
|
3364
|
+
`Delayed relay to "${target}" failed \u2014 message will be redelivered:`,
|
|
3365
|
+
toError(txErr).stack
|
|
3366
|
+
);
|
|
3367
|
+
}
|
|
3368
|
+
}
|
|
3369
|
+
});
|
|
3370
|
+
ctx.runningConsumers.set(gid, "eachMessage");
|
|
3371
|
+
ctx.logger.log(
|
|
3372
|
+
`Delayed relay started for: ${delayedTopics.join(", ")} (group: ${gid})`
|
|
3373
|
+
);
|
|
3374
|
+
return {
|
|
3375
|
+
groupId: gid,
|
|
3376
|
+
ready: () => readyPromise,
|
|
3377
|
+
stop: async () => {
|
|
3378
|
+
await stopConsumerImpl(ctx, gid);
|
|
3379
|
+
}
|
|
3380
|
+
};
|
|
3381
|
+
}
|
|
3382
|
+
|
|
3383
|
+
// src/client/kafka.client/consumer/features/snapshot.ts
|
|
3384
|
+
async function readSnapshotImpl(ctx, topic, options = {}) {
|
|
2967
3385
|
await ctx.adminOps.ensureConnected();
|
|
2968
3386
|
let offsets;
|
|
2969
3387
|
try {
|
|
2970
|
-
offsets = await ctx.adminOps.admin.fetchTopicOffsets(
|
|
3388
|
+
offsets = await ctx.adminOps.admin.fetchTopicOffsets(topic);
|
|
2971
3389
|
} catch {
|
|
2972
3390
|
ctx.logger.warn(
|
|
2973
|
-
`readSnapshot: could not fetch offsets for "${String(
|
|
3391
|
+
`readSnapshot: could not fetch offsets for "${String(topic)}", returning empty snapshot`
|
|
2974
3392
|
);
|
|
2975
3393
|
return /* @__PURE__ */ new Map();
|
|
2976
3394
|
}
|
|
@@ -2982,7 +3400,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
|
|
|
2982
3400
|
}
|
|
2983
3401
|
if (targets.size === 0) {
|
|
2984
3402
|
ctx.logger.debug?.(
|
|
2985
|
-
`readSnapshot: topic "${String(
|
|
3403
|
+
`readSnapshot: topic "${String(topic)}" is empty \u2014 returning empty snapshot`
|
|
2986
3404
|
);
|
|
2987
3405
|
return /* @__PURE__ */ new Map();
|
|
2988
3406
|
}
|
|
@@ -3001,7 +3419,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
|
|
|
3001
3419
|
});
|
|
3002
3420
|
});
|
|
3003
3421
|
};
|
|
3004
|
-
consumer.connect().then(() => consumer.subscribe({ topics: [
|
|
3422
|
+
consumer.connect().then(() => consumer.subscribe({ topics: [topic] })).then(
|
|
3005
3423
|
() => consumer.run({
|
|
3006
3424
|
eachMessage: async ({ topic: t, partition, message }) => {
|
|
3007
3425
|
if (!remaining.has(partition)) return;
|
|
@@ -3022,7 +3440,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
|
|
|
3022
3440
|
});
|
|
3023
3441
|
});
|
|
3024
3442
|
ctx.logger.log(
|
|
3025
|
-
`readSnapshot: ${snapshot.size} key(s) from "${String(
|
|
3443
|
+
`readSnapshot: ${snapshot.size} key(s) from "${String(topic)}"`
|
|
3026
3444
|
);
|
|
3027
3445
|
return snapshot;
|
|
3028
3446
|
}
|
|
@@ -3059,9 +3477,9 @@ async function checkpointOffsetsImpl(ctx, groupId, checkpointTopic) {
|
|
|
3059
3477
|
await ctx.adminOps.ensureConnected();
|
|
3060
3478
|
const committed = await ctx.adminOps.admin.fetchOffsets({ groupId: gid });
|
|
3061
3479
|
const offsets = [];
|
|
3062
|
-
for (const { topic
|
|
3480
|
+
for (const { topic, partitions } of committed) {
|
|
3063
3481
|
for (const { partition, offset } of partitions) {
|
|
3064
|
-
offsets.push({ topic
|
|
3482
|
+
offsets.push({ topic, partition, offset });
|
|
3065
3483
|
}
|
|
3066
3484
|
}
|
|
3067
3485
|
const savedAt = Date.now();
|
|
@@ -3228,6 +3646,7 @@ var KafkaClient = class {
|
|
|
3228
3646
|
* ```
|
|
3229
3647
|
*/
|
|
3230
3648
|
constructor(clientId, groupId, brokers, options) {
|
|
3649
|
+
validateClientOptions(clientId, groupId, brokers, options);
|
|
3231
3650
|
this.clientId = clientId;
|
|
3232
3651
|
const logger = options?.logger ?? {
|
|
3233
3652
|
log: (msg) => console.log(`[KafkaClient:${clientId}] ${msg}`),
|
|
@@ -3235,12 +3654,14 @@ var KafkaClient = class {
|
|
|
3235
3654
|
error: (msg, ...args) => console.error(`[KafkaClient:${clientId}] ${msg}`, ...args),
|
|
3236
3655
|
debug: (msg, ...args) => console.debug(`[KafkaClient:${clientId}] ${msg}`, ...args)
|
|
3237
3656
|
};
|
|
3238
|
-
const
|
|
3657
|
+
const security = resolveSecurityOptions(options?.security, brokers, logger);
|
|
3658
|
+
const transport = options?.transport ?? new ConfluentTransport(clientId, brokers, security);
|
|
3239
3659
|
const producer = transport.producer();
|
|
3240
3660
|
const runningConsumers = /* @__PURE__ */ new Map();
|
|
3241
3661
|
const consumers = /* @__PURE__ */ new Map();
|
|
3242
3662
|
const consumerCreationOptions = /* @__PURE__ */ new Map();
|
|
3243
3663
|
const schemaRegistry = /* @__PURE__ */ new Map();
|
|
3664
|
+
const serde = options?.serde ?? new JsonSerde();
|
|
3244
3665
|
const adminOps = new AdminOps({
|
|
3245
3666
|
admin: transport.admin(),
|
|
3246
3667
|
logger,
|
|
@@ -3267,8 +3688,10 @@ var KafkaClient = class {
|
|
|
3267
3688
|
autoCreateTopicsEnabled: options?.autoCreateTopics ?? false,
|
|
3268
3689
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3269
3690
|
numPartitions: options?.numPartitions ?? 1,
|
|
3691
|
+
serde,
|
|
3270
3692
|
txId: options?.transactionalId ?? `${clientId}-tx`,
|
|
3271
3693
|
clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
|
|
3694
|
+
clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
|
|
3272
3695
|
lagThrottleOpts: options?.lagThrottle,
|
|
3273
3696
|
instrumentation: options?.instrumentation ?? [],
|
|
3274
3697
|
onMessageLost: options?.onMessageLost,
|
|
@@ -3278,6 +3701,7 @@ var KafkaClient = class {
|
|
|
3278
3701
|
producer,
|
|
3279
3702
|
txProducer: void 0,
|
|
3280
3703
|
txProducerInitPromise: void 0,
|
|
3704
|
+
_txChain: Promise.resolve(),
|
|
3281
3705
|
retryTxProducers: /* @__PURE__ */ new Map(),
|
|
3282
3706
|
consumers,
|
|
3283
3707
|
runningConsumers,
|
|
@@ -3299,6 +3723,7 @@ var KafkaClient = class {
|
|
|
3299
3723
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3300
3724
|
instrumentation: options?.instrumentation ?? [],
|
|
3301
3725
|
logger,
|
|
3726
|
+
serde,
|
|
3302
3727
|
nextLamportClock: () => 0
|
|
3303
3728
|
// patched below
|
|
3304
3729
|
},
|
|
@@ -3317,6 +3742,7 @@ var KafkaClient = class {
|
|
|
3317
3742
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3318
3743
|
instrumentation: options?.instrumentation ?? [],
|
|
3319
3744
|
logger,
|
|
3745
|
+
serde,
|
|
3320
3746
|
nextLamportClock: () => ++ctx._lamportClock
|
|
3321
3747
|
};
|
|
3322
3748
|
ctx.retryTopicDeps = buildRetryTopicDeps(ctx);
|
|
@@ -3326,8 +3752,8 @@ var KafkaClient = class {
|
|
|
3326
3752
|
return sendMessageImpl(this.ctx, topicOrDesc, message, options);
|
|
3327
3753
|
}
|
|
3328
3754
|
/** @inheritDoc */
|
|
3329
|
-
async sendTombstone(
|
|
3330
|
-
return sendTombstoneImpl(this.ctx,
|
|
3755
|
+
async sendTombstone(topic, key, headers) {
|
|
3756
|
+
return sendTombstoneImpl(this.ctx, topic, key, headers);
|
|
3331
3757
|
}
|
|
3332
3758
|
async sendBatch(topicOrDesc, messages, options) {
|
|
3333
3759
|
return sendBatchImpl(this.ctx, topicOrDesc, messages, options);
|
|
@@ -3358,7 +3784,7 @@ var KafkaClient = class {
|
|
|
3358
3784
|
}
|
|
3359
3785
|
// ── Consumer: AsyncIterableIterator ──────────────────────────────
|
|
3360
3786
|
/** @inheritDoc */
|
|
3361
|
-
consume(
|
|
3787
|
+
consume(topic, options) {
|
|
3362
3788
|
if (options?.retryTopics) {
|
|
3363
3789
|
throw new Error(
|
|
3364
3790
|
"consume() does not support retryTopics (EOS retry chains). Use startConsumer() with retryTopics: true for guaranteed retry delivery."
|
|
@@ -3367,11 +3793,11 @@ var KafkaClient = class {
|
|
|
3367
3793
|
const gid = options?.groupId ?? this.ctx.defaultGroupId;
|
|
3368
3794
|
const queue = new AsyncQueue(
|
|
3369
3795
|
options?.queueHighWaterMark,
|
|
3370
|
-
() => pauseTopicAllPartitions(this.ctx, gid,
|
|
3371
|
-
() => resumeTopicAllPartitions(this.ctx, gid,
|
|
3796
|
+
() => pauseTopicAllPartitions(this.ctx, gid, topic),
|
|
3797
|
+
() => resumeTopicAllPartitions(this.ctx, gid, topic)
|
|
3372
3798
|
);
|
|
3373
3799
|
const handlePromise = this.startConsumer(
|
|
3374
|
-
[
|
|
3800
|
+
[topic],
|
|
3375
3801
|
async (envelope) => {
|
|
3376
3802
|
queue.push(envelope);
|
|
3377
3803
|
},
|
|
@@ -3393,14 +3819,39 @@ var KafkaClient = class {
|
|
|
3393
3819
|
}
|
|
3394
3820
|
// ── Consumer: windowed ────────────────────────────────────────────
|
|
3395
3821
|
/** @inheritDoc */
|
|
3396
|
-
startWindowConsumer(
|
|
3397
|
-
return startWindowConsumerImpl(this.ctx,
|
|
3822
|
+
startWindowConsumer(topic, handler, options) {
|
|
3823
|
+
return startWindowConsumerImpl(this.ctx, topic, handler, options);
|
|
3398
3824
|
}
|
|
3399
3825
|
// ── Consumer: header routing ──────────────────────────────────────
|
|
3400
3826
|
/** @inheritDoc */
|
|
3401
3827
|
startRoutedConsumer(topics, routing, options) {
|
|
3402
3828
|
return startRoutedConsumerImpl(this.ctx, topics, routing, options);
|
|
3403
3829
|
}
|
|
3830
|
+
// ── Consumer: delayed delivery relay ──────────────────────────────
|
|
3831
|
+
/**
|
|
3832
|
+
* Start a relay that delivers messages produced with
|
|
3833
|
+
* `SendOptions.deliverAfterMs` from `<topic>.delayed` to their target topic
|
|
3834
|
+
* once their deadline passes.
|
|
3835
|
+
*
|
|
3836
|
+
* Forwarding is transactional (produce + source-offset commit are atomic),
|
|
3837
|
+
* so no duplicates are relayed even if the relay crashes mid-forward.
|
|
3838
|
+
* Delivery time is a lower bound — the relay must be running for delayed
|
|
3839
|
+
* messages to be delivered at all.
|
|
3840
|
+
*
|
|
3841
|
+
* @param topics Target topic name(s) whose `<topic>.delayed` staging topics to relay.
|
|
3842
|
+
* @param options Optional `groupId` override (default: `<defaultGroupId>-delayed-relay`).
|
|
3843
|
+
*
|
|
3844
|
+
* @example
|
|
3845
|
+
* ```ts
|
|
3846
|
+
* await kafka.startDelayedRelay(['orders.reminder']);
|
|
3847
|
+
* await kafka.sendMessage('orders.reminder', payload, { deliverAfterMs: 60_000 });
|
|
3848
|
+
* // → delivered to orders.reminder ~60 s later
|
|
3849
|
+
* ```
|
|
3850
|
+
*/
|
|
3851
|
+
async startDelayedRelay(topics, options) {
|
|
3852
|
+
const list = Array.isArray(topics) ? topics : [topics];
|
|
3853
|
+
return startDelayedRelayImpl(this.ctx, list, options);
|
|
3854
|
+
}
|
|
3404
3855
|
// ── Consumer: transactional EOS ───────────────────────────────────
|
|
3405
3856
|
/** @inheritDoc */
|
|
3406
3857
|
async startTransactionalConsumer(topics, handler, options = {}) {
|
|
@@ -3421,10 +3872,10 @@ var KafkaClient = class {
|
|
|
3421
3872
|
}
|
|
3422
3873
|
// ── DLQ replay ────────────────────────────────────────────────────
|
|
3423
3874
|
/** @inheritDoc */
|
|
3424
|
-
async replayDlq(
|
|
3875
|
+
async replayDlq(topic, options = {}) {
|
|
3425
3876
|
await this.ctx.adminOps.ensureConnected();
|
|
3426
3877
|
return replayDlqTopic(
|
|
3427
|
-
|
|
3878
|
+
topic,
|
|
3428
3879
|
{
|
|
3429
3880
|
logger: this.ctx.logger,
|
|
3430
3881
|
fetchTopicOffsets: (t) => this.ctx.adminOps.admin.fetchTopicOffsets(t),
|
|
@@ -3451,8 +3902,8 @@ var KafkaClient = class {
|
|
|
3451
3902
|
}
|
|
3452
3903
|
// ── Snapshot & checkpoint ─────────────────────────────────────────
|
|
3453
3904
|
/** @inheritDoc */
|
|
3454
|
-
async readSnapshot(
|
|
3455
|
-
return readSnapshotImpl(this.ctx,
|
|
3905
|
+
async readSnapshot(topic, options = {}) {
|
|
3906
|
+
return readSnapshotImpl(this.ctx, topic, options);
|
|
3456
3907
|
}
|
|
3457
3908
|
/** @inheritDoc */
|
|
3458
3909
|
async checkpointOffsets(groupId, checkpointTopic) {
|
|
@@ -3464,8 +3915,8 @@ var KafkaClient = class {
|
|
|
3464
3915
|
}
|
|
3465
3916
|
// ── Admin ─────────────────────────────────────────────────────────
|
|
3466
3917
|
/** @inheritDoc */
|
|
3467
|
-
async resetOffsets(groupId,
|
|
3468
|
-
return this.ctx.adminOps.resetOffsets(groupId,
|
|
3918
|
+
async resetOffsets(groupId, topic, position) {
|
|
3919
|
+
return this.ctx.adminOps.resetOffsets(groupId, topic, position);
|
|
3469
3920
|
}
|
|
3470
3921
|
/** @inheritDoc */
|
|
3471
3922
|
async seekToOffset(groupId, assignments) {
|
|
@@ -3492,26 +3943,26 @@ var KafkaClient = class {
|
|
|
3492
3943
|
return this.ctx.adminOps.describeTopics(topics);
|
|
3493
3944
|
}
|
|
3494
3945
|
/** @inheritDoc */
|
|
3495
|
-
async deleteRecords(
|
|
3496
|
-
return this.ctx.adminOps.deleteRecords(
|
|
3946
|
+
async deleteRecords(topic, partitions) {
|
|
3947
|
+
return this.ctx.adminOps.deleteRecords(topic, partitions);
|
|
3497
3948
|
}
|
|
3498
3949
|
// ── Circuit breaker ───────────────────────────────────────────────
|
|
3499
3950
|
/** @inheritDoc */
|
|
3500
|
-
getCircuitState(
|
|
3951
|
+
getCircuitState(topic, partition, groupId) {
|
|
3501
3952
|
return this.ctx.circuitBreaker.getState(
|
|
3502
|
-
|
|
3953
|
+
topic,
|
|
3503
3954
|
partition,
|
|
3504
3955
|
groupId ?? this.ctx.defaultGroupId
|
|
3505
3956
|
);
|
|
3506
3957
|
}
|
|
3507
3958
|
// ── Metrics ───────────────────────────────────────────────────────
|
|
3508
3959
|
/** @inheritDoc */
|
|
3509
|
-
getMetrics(
|
|
3510
|
-
return this.ctx.metrics.getMetrics(
|
|
3960
|
+
getMetrics(topic) {
|
|
3961
|
+
return this.ctx.metrics.getMetrics(topic);
|
|
3511
3962
|
}
|
|
3512
3963
|
/** @inheritDoc */
|
|
3513
|
-
resetMetrics(
|
|
3514
|
-
this.ctx.metrics.resetMetrics(
|
|
3964
|
+
resetMetrics(topic) {
|
|
3965
|
+
this.ctx.metrics.resetMetrics(topic);
|
|
3515
3966
|
}
|
|
3516
3967
|
getClientId() {
|
|
3517
3968
|
return this.clientId;
|
|
@@ -3542,38 +3993,351 @@ var KafkaClient = class {
|
|
|
3542
3993
|
}
|
|
3543
3994
|
};
|
|
3544
3995
|
|
|
3545
|
-
// src/
|
|
3546
|
-
|
|
3996
|
+
// src/cli/dlq.ts
|
|
3997
|
+
var DLQ_SUFFIX = ".dlq";
|
|
3998
|
+
var DlqUsageError = class extends Error {
|
|
3999
|
+
constructor(message) {
|
|
4000
|
+
super(message);
|
|
4001
|
+
this.name = "DlqUsageError";
|
|
4002
|
+
}
|
|
4003
|
+
};
|
|
4004
|
+
var USAGE = `kafka-client-dlq \u2014 dead-letter queue operations
|
|
4005
|
+
|
|
4006
|
+
Usage:
|
|
4007
|
+
kafka-client-dlq ls --brokers <b1,b2> [--prefix <name>]
|
|
4008
|
+
kafka-client-dlq peek --brokers <b1,b2> --topic <name> [--limit <n>]
|
|
4009
|
+
kafka-client-dlq replay --brokers <b1,b2> --topic <name> [--target <t>] [--dry-run] [--from-beginning | --incremental]
|
|
4010
|
+
|
|
4011
|
+
Commands:
|
|
4012
|
+
ls List DLQ topics (ending in .dlq) with per-topic message counts.
|
|
4013
|
+
peek Print up to N messages from <topic>.dlq (offset, x-dlq-* headers, value).
|
|
4014
|
+
replay Re-publish <topic>.dlq messages to their original topic (or --target).
|
|
4015
|
+
|
|
4016
|
+
Options:
|
|
4017
|
+
--brokers <list> Comma-separated broker addresses (required). e.g. localhost:9092
|
|
4018
|
+
--prefix <name> ls: only show DLQ topics whose base name starts with <name>.
|
|
4019
|
+
--topic <name> peek/replay: base topic name (the CLI uses <name>.dlq).
|
|
4020
|
+
--limit <n> peek: max messages to print (default 10).
|
|
4021
|
+
--target <t> replay: override destination topic.
|
|
4022
|
+
--dry-run replay: log without publishing.
|
|
4023
|
+
--from-beginning replay: full replay every call (default).
|
|
4024
|
+
--incremental replay: only messages added since the previous replay.
|
|
4025
|
+
-h, --help Show this help.
|
|
4026
|
+
|
|
4027
|
+
Examples:
|
|
4028
|
+
kafka-client-dlq ls --brokers localhost:9092
|
|
4029
|
+
kafka-client-dlq ls --brokers localhost:9092 --prefix orders
|
|
4030
|
+
kafka-client-dlq peek --brokers localhost:9092 --topic orders.created --limit 5
|
|
4031
|
+
kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --dry-run
|
|
4032
|
+
kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --target orders.manual --incremental
|
|
4033
|
+
`;
|
|
4034
|
+
var VALUE_FLAGS = /* @__PURE__ */ new Set([
|
|
4035
|
+
"--brokers",
|
|
4036
|
+
"--prefix",
|
|
4037
|
+
"--topic",
|
|
4038
|
+
"--limit",
|
|
4039
|
+
"--target"
|
|
4040
|
+
]);
|
|
4041
|
+
var BOOL_FLAGS = /* @__PURE__ */ new Set(["--dry-run", "--from-beginning", "--incremental"]);
|
|
4042
|
+
function parseFlags(args) {
|
|
4043
|
+
const values = {};
|
|
4044
|
+
const bools = /* @__PURE__ */ new Set();
|
|
4045
|
+
for (let i = 0; i < args.length; i++) {
|
|
4046
|
+
const arg = args[i];
|
|
4047
|
+
if (!arg.startsWith("--")) {
|
|
4048
|
+
throw new DlqUsageError(`Unexpected argument: "${arg}"`);
|
|
4049
|
+
}
|
|
4050
|
+
if (VALUE_FLAGS.has(arg)) {
|
|
4051
|
+
const value = args[i + 1];
|
|
4052
|
+
if (value === void 0 || value.startsWith("--")) {
|
|
4053
|
+
throw new DlqUsageError(`Flag "${arg}" requires a value.`);
|
|
4054
|
+
}
|
|
4055
|
+
values[arg] = value;
|
|
4056
|
+
i++;
|
|
4057
|
+
} else if (BOOL_FLAGS.has(arg)) {
|
|
4058
|
+
bools.add(arg);
|
|
4059
|
+
} else {
|
|
4060
|
+
throw new DlqUsageError(`Unknown flag: "${arg}"`);
|
|
4061
|
+
}
|
|
4062
|
+
}
|
|
4063
|
+
return { values, bools };
|
|
4064
|
+
}
|
|
4065
|
+
function requireBrokers(flags) {
|
|
4066
|
+
const raw = flags.values["--brokers"];
|
|
4067
|
+
if (raw === void 0) {
|
|
4068
|
+
throw new DlqUsageError("Missing required flag: --brokers");
|
|
4069
|
+
}
|
|
4070
|
+
const brokers = raw.split(",").map((b) => b.trim()).filter((b) => b.length > 0);
|
|
4071
|
+
if (brokers.length === 0) {
|
|
4072
|
+
throw new DlqUsageError("--brokers must list at least one broker address.");
|
|
4073
|
+
}
|
|
4074
|
+
return brokers;
|
|
4075
|
+
}
|
|
4076
|
+
function requireTopic(flags) {
|
|
4077
|
+
const topic = flags.values["--topic"];
|
|
4078
|
+
if (topic === void 0 || topic.length === 0) {
|
|
4079
|
+
throw new DlqUsageError("Missing required flag: --topic");
|
|
4080
|
+
}
|
|
4081
|
+
return topic;
|
|
4082
|
+
}
|
|
4083
|
+
function parseArgs(argv) {
|
|
4084
|
+
const [command, ...rest] = argv;
|
|
4085
|
+
if (command === void 0 || command === "-h" || command === "--help" || command === "help") {
|
|
4086
|
+
return { command: "help" };
|
|
4087
|
+
}
|
|
4088
|
+
switch (command) {
|
|
4089
|
+
case "ls": {
|
|
4090
|
+
const flags = parseFlags(rest);
|
|
4091
|
+
const brokers = requireBrokers(flags);
|
|
4092
|
+
const prefix = flags.values["--prefix"];
|
|
4093
|
+
return { command: "ls", brokers, prefix };
|
|
4094
|
+
}
|
|
4095
|
+
case "peek": {
|
|
4096
|
+
const flags = parseFlags(rest);
|
|
4097
|
+
const brokers = requireBrokers(flags);
|
|
4098
|
+
const topic = requireTopic(flags);
|
|
4099
|
+
const limit = parseLimit(flags.values["--limit"]);
|
|
4100
|
+
return { command: "peek", brokers, topic, limit };
|
|
4101
|
+
}
|
|
4102
|
+
case "replay": {
|
|
4103
|
+
const flags = parseFlags(rest);
|
|
4104
|
+
const brokers = requireBrokers(flags);
|
|
4105
|
+
const topic = requireTopic(flags);
|
|
4106
|
+
const target = flags.values["--target"];
|
|
4107
|
+
const dryRun = flags.bools.has("--dry-run");
|
|
4108
|
+
if (flags.bools.has("--from-beginning") && flags.bools.has("--incremental")) {
|
|
4109
|
+
throw new DlqUsageError(
|
|
4110
|
+
"--from-beginning and --incremental are mutually exclusive."
|
|
4111
|
+
);
|
|
4112
|
+
}
|
|
4113
|
+
const fromBeginning = !flags.bools.has("--incremental");
|
|
4114
|
+
return { command: "replay", brokers, topic, target, dryRun, fromBeginning };
|
|
4115
|
+
}
|
|
4116
|
+
default:
|
|
4117
|
+
throw new DlqUsageError(`Unknown command: "${command}"`);
|
|
4118
|
+
}
|
|
4119
|
+
}
|
|
4120
|
+
function parseLimit(raw) {
|
|
4121
|
+
if (raw === void 0) return 10;
|
|
4122
|
+
const n = Number(raw);
|
|
4123
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
4124
|
+
throw new DlqUsageError(`--limit must be a positive integer, got "${raw}".`);
|
|
4125
|
+
}
|
|
4126
|
+
return n;
|
|
4127
|
+
}
|
|
4128
|
+
function countFromWatermarks(watermarks) {
|
|
4129
|
+
let total = 0;
|
|
4130
|
+
for (const { low, high } of watermarks) {
|
|
4131
|
+
const width = Number(high) - Number(low);
|
|
4132
|
+
total += width > 0 ? width : 0;
|
|
4133
|
+
}
|
|
4134
|
+
return total;
|
|
4135
|
+
}
|
|
4136
|
+
function truncate(value, max = 200) {
|
|
4137
|
+
if (value.length <= max) return value;
|
|
4138
|
+
return `${value.slice(0, max)}\u2026 (${value.length} chars)`;
|
|
4139
|
+
}
|
|
4140
|
+
async function runDlqCommand(cmd, deps) {
|
|
4141
|
+
if (cmd.command === "help") {
|
|
4142
|
+
deps.out(USAGE);
|
|
4143
|
+
return { command: "help" };
|
|
4144
|
+
}
|
|
4145
|
+
const client = await deps.createClient(cmd.brokers);
|
|
4146
|
+
try {
|
|
4147
|
+
switch (cmd.command) {
|
|
4148
|
+
case "ls":
|
|
4149
|
+
return await runLs(cmd, client, deps);
|
|
4150
|
+
case "peek":
|
|
4151
|
+
return await runPeek(cmd, client, deps);
|
|
4152
|
+
case "replay":
|
|
4153
|
+
return await runReplay(cmd, client, deps);
|
|
4154
|
+
}
|
|
4155
|
+
} finally {
|
|
4156
|
+
await client.close();
|
|
4157
|
+
}
|
|
4158
|
+
}
|
|
4159
|
+
async function runLs(cmd, client, deps) {
|
|
4160
|
+
const allTopics = await client.listTopics();
|
|
4161
|
+
let dlqTopics = allTopics.filter((t) => t.endsWith(DLQ_SUFFIX));
|
|
4162
|
+
if (cmd.prefix) {
|
|
4163
|
+
const prefix = cmd.prefix;
|
|
4164
|
+
dlqTopics = dlqTopics.filter(
|
|
4165
|
+
(t) => t.slice(0, -DLQ_SUFFIX.length).startsWith(prefix)
|
|
4166
|
+
);
|
|
4167
|
+
}
|
|
4168
|
+
dlqTopics.sort();
|
|
4169
|
+
const counts = [];
|
|
4170
|
+
for (const dlqTopic of dlqTopics) {
|
|
4171
|
+
const watermarks = await client.fetchTopicOffsets(dlqTopic);
|
|
4172
|
+
counts.push({
|
|
4173
|
+
dlqTopic,
|
|
4174
|
+
baseTopic: dlqTopic.slice(0, -DLQ_SUFFIX.length),
|
|
4175
|
+
count: countFromWatermarks(watermarks)
|
|
4176
|
+
});
|
|
4177
|
+
}
|
|
4178
|
+
if (counts.length === 0) {
|
|
4179
|
+
deps.out(
|
|
4180
|
+
cmd.prefix ? `No DLQ topics found matching prefix "${cmd.prefix}".` : "No DLQ topics found."
|
|
4181
|
+
);
|
|
4182
|
+
} else {
|
|
4183
|
+
const width = Math.max(...counts.map((c) => c.dlqTopic.length));
|
|
4184
|
+
deps.out(`${"TOPIC".padEnd(width)} MESSAGES`);
|
|
4185
|
+
for (const c of counts) {
|
|
4186
|
+
deps.out(`${c.dlqTopic.padEnd(width)} ${c.count}`);
|
|
4187
|
+
}
|
|
4188
|
+
const total = counts.reduce((s, c) => s + c.count, 0);
|
|
4189
|
+
deps.out(`${counts.length} DLQ topic(s), ${total} message(s) total.`);
|
|
4190
|
+
}
|
|
4191
|
+
return { command: "ls", topics: counts };
|
|
4192
|
+
}
|
|
4193
|
+
async function runPeek(cmd, client, deps) {
|
|
4194
|
+
const dlqTopic = `${cmd.topic}${DLQ_SUFFIX}`;
|
|
4195
|
+
const messages = await client.peekMessages(dlqTopic, cmd.limit);
|
|
4196
|
+
if (messages.length === 0) {
|
|
4197
|
+
deps.out(`No messages in ${dlqTopic}.`);
|
|
4198
|
+
return { command: "peek", printed: 0 };
|
|
4199
|
+
}
|
|
4200
|
+
deps.out(`Peeking up to ${cmd.limit} message(s) from ${dlqTopic}:`);
|
|
4201
|
+
let printed = 0;
|
|
4202
|
+
for (const env of messages) {
|
|
4203
|
+
if (printed >= cmd.limit) break;
|
|
4204
|
+
deps.out("");
|
|
4205
|
+
deps.out(
|
|
4206
|
+
`\u2500 offset ${env.offset} \xB7 partition ${env.partition} \xB7 ${env.timestamp}`
|
|
4207
|
+
);
|
|
4208
|
+
const dlqHeaders = Object.entries(env.headers).filter(([k]) => k.startsWith("x-dlq-")).sort(([a], [b]) => a.localeCompare(b));
|
|
4209
|
+
for (const [k, v] of dlqHeaders) {
|
|
4210
|
+
deps.out(` ${k}: ${truncate(String(v), 500)}`);
|
|
4211
|
+
}
|
|
4212
|
+
deps.out(` value: ${truncate(JSON.stringify(env.payload))}`);
|
|
4213
|
+
printed++;
|
|
4214
|
+
}
|
|
4215
|
+
deps.out("");
|
|
4216
|
+
deps.out(`Printed ${printed} message(s).`);
|
|
4217
|
+
return { command: "peek", printed };
|
|
4218
|
+
}
|
|
4219
|
+
async function runReplay(cmd, client, deps) {
|
|
4220
|
+
const options = {
|
|
4221
|
+
dryRun: cmd.dryRun,
|
|
4222
|
+
fromBeginning: cmd.fromBeginning
|
|
4223
|
+
};
|
|
4224
|
+
if (cmd.target !== void 0) options.targetTopic = cmd.target;
|
|
4225
|
+
const mode = cmd.fromBeginning ? "full" : "incremental";
|
|
4226
|
+
const targetDesc = cmd.target ? ` \u2192 ${cmd.target}` : " \u2192 original topic";
|
|
4227
|
+
deps.out(
|
|
4228
|
+
`Replaying ${cmd.topic}${DLQ_SUFFIX}${targetDesc} (${mode}${cmd.dryRun ? ", dry-run" : ""})\u2026`
|
|
4229
|
+
);
|
|
4230
|
+
const { replayed, skipped } = await client.replayDlq(cmd.topic, options);
|
|
4231
|
+
deps.out(
|
|
4232
|
+
cmd.dryRun ? `Dry-run: ${replayed} message(s) would be replayed, ${skipped} skipped.` : `Replayed ${replayed} message(s), ${skipped} skipped.`
|
|
4233
|
+
);
|
|
4234
|
+
return { command: "replay", replayed, skipped, dryRun: cmd.dryRun };
|
|
4235
|
+
}
|
|
4236
|
+
|
|
4237
|
+
// src/cli/index.ts
|
|
4238
|
+
var CLIENT_ID = `dlq-cli-${process.pid}`;
|
|
4239
|
+
var GROUP_ID = `${CLIENT_ID}-peek`;
|
|
4240
|
+
function createRealClient(brokers) {
|
|
4241
|
+
const transport = new ConfluentTransport(CLIENT_ID, brokers);
|
|
4242
|
+
const kafka = new KafkaClient(
|
|
4243
|
+
CLIENT_ID,
|
|
4244
|
+
GROUP_ID,
|
|
4245
|
+
brokers,
|
|
4246
|
+
{ transport, autoCreateTopics: false, strictSchemas: false }
|
|
4247
|
+
);
|
|
4248
|
+
const admin = transport.admin();
|
|
4249
|
+
let adminConnected = false;
|
|
4250
|
+
async function ensureAdmin() {
|
|
4251
|
+
if (!adminConnected) {
|
|
4252
|
+
await admin.connect();
|
|
4253
|
+
adminConnected = true;
|
|
4254
|
+
}
|
|
4255
|
+
}
|
|
3547
4256
|
return {
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
4257
|
+
async listTopics() {
|
|
4258
|
+
const status = await kafka.checkStatus();
|
|
4259
|
+
if (status.status === "down") {
|
|
4260
|
+
throw new Error(`Broker unreachable: ${status.error}`);
|
|
4261
|
+
}
|
|
4262
|
+
return status.topics;
|
|
4263
|
+
},
|
|
4264
|
+
async fetchTopicOffsets(topic) {
|
|
4265
|
+
await ensureAdmin();
|
|
4266
|
+
return admin.fetchTopicOffsets(topic);
|
|
4267
|
+
},
|
|
4268
|
+
async peekMessages(dlqTopic, limit) {
|
|
4269
|
+
const collected = [];
|
|
4270
|
+
const iterator = kafka.consume(dlqTopic, {
|
|
4271
|
+
groupId: `${dlqTopic}.dlq-peek-${Date.now()}`,
|
|
4272
|
+
fromBeginning: true
|
|
4273
|
+
});
|
|
4274
|
+
await ensureAdmin();
|
|
4275
|
+
const watermarks = await admin.fetchTopicOffsets(dlqTopic);
|
|
4276
|
+
const available = watermarks.reduce(
|
|
4277
|
+
(sum, w) => sum + Math.max(0, Number(w.high) - Number(w.low)),
|
|
4278
|
+
0
|
|
4279
|
+
);
|
|
4280
|
+
const target = Math.min(limit, available);
|
|
4281
|
+
if (target === 0) {
|
|
4282
|
+
await iterator.return?.();
|
|
4283
|
+
return collected;
|
|
4284
|
+
}
|
|
4285
|
+
try {
|
|
4286
|
+
for await (const env of iterator) {
|
|
4287
|
+
collected.push(env);
|
|
4288
|
+
if (collected.length >= target) break;
|
|
4289
|
+
}
|
|
4290
|
+
} finally {
|
|
4291
|
+
await iterator.return?.();
|
|
4292
|
+
}
|
|
4293
|
+
return collected;
|
|
4294
|
+
},
|
|
4295
|
+
replayDlq(topic, options) {
|
|
4296
|
+
return kafka.replayDlq(topic, options);
|
|
4297
|
+
},
|
|
4298
|
+
async close() {
|
|
4299
|
+
if (adminConnected) {
|
|
4300
|
+
await admin.disconnect().catch(() => {
|
|
4301
|
+
});
|
|
4302
|
+
}
|
|
4303
|
+
await kafka.disconnect().catch(() => {
|
|
4304
|
+
});
|
|
4305
|
+
}
|
|
3558
4306
|
};
|
|
3559
4307
|
}
|
|
4308
|
+
async function main() {
|
|
4309
|
+
let cmd;
|
|
4310
|
+
try {
|
|
4311
|
+
cmd = parseArgs(process.argv.slice(2));
|
|
4312
|
+
} catch (err) {
|
|
4313
|
+
if (err instanceof DlqUsageError) {
|
|
4314
|
+
process.stderr.write(`Error: ${err.message}
|
|
3560
4315
|
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
4316
|
+
`);
|
|
4317
|
+
process.stderr.write(USAGE);
|
|
4318
|
+
return 2;
|
|
4319
|
+
}
|
|
4320
|
+
throw err;
|
|
4321
|
+
}
|
|
4322
|
+
try {
|
|
4323
|
+
await runDlqCommand(cmd, {
|
|
4324
|
+
createClient: createRealClient,
|
|
4325
|
+
out: (line) => process.stdout.write(`${line}
|
|
4326
|
+
`)
|
|
4327
|
+
});
|
|
4328
|
+
return 0;
|
|
4329
|
+
} catch (err) {
|
|
4330
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4331
|
+
process.stderr.write(`Error: ${message}
|
|
4332
|
+
`);
|
|
4333
|
+
return 1;
|
|
4334
|
+
}
|
|
4335
|
+
}
|
|
4336
|
+
main().then((code) => {
|
|
4337
|
+
process.exitCode = code;
|
|
4338
|
+
}).catch((err) => {
|
|
4339
|
+
process.stderr.write(`Fatal: ${err?.stack ?? err}
|
|
4340
|
+
`);
|
|
4341
|
+
process.exitCode = 1;
|
|
4342
|
+
});
|
|
4343
|
+
//# sourceMappingURL=index.js.map
|