@drarzter/kafka-client 0.7.2 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -7
- package/dist/{chunk-MADAJD2F.mjs → chunk-BVWRZTMD.mjs} +858 -864
- package/dist/chunk-BVWRZTMD.mjs.map +1 -0
- package/dist/core.d.mts +36 -139
- package/dist/core.d.ts +36 -139
- package/dist/core.js +857 -863
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +853 -859
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/otel.d.ts +1 -1
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/{types-4qWrf2aJ.d.mts → types-Db7qSbZP.d.mts} +14 -0
- package/dist/{types-4qWrf2aJ.d.ts → types-Db7qSbZP.d.ts} +14 -0
- package/package.json +1 -1
- package/dist/chunk-MADAJD2F.mjs.map +0 -1
|
@@ -95,7 +95,7 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
|
95
95
|
}
|
|
96
96
|
};
|
|
97
97
|
|
|
98
|
-
// src/client/kafka.client/producer
|
|
98
|
+
// src/client/kafka.client/producer/ops.ts
|
|
99
99
|
function resolveTopicName(topicOrDescriptor) {
|
|
100
100
|
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
101
101
|
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
@@ -173,7 +173,7 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
173
173
|
return { topic: topic2, messages: builtMessages, ...compression && { compression } };
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
// src/client/kafka.client/consumer
|
|
176
|
+
// src/client/kafka.client/consumer/ops.ts
|
|
177
177
|
import { KafkaJS } from "@confluentinc/kafka-javascript";
|
|
178
178
|
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner) {
|
|
179
179
|
const { consumers, consumerCreationOptions, kafka, onRebalance, logger } = deps;
|
|
@@ -236,7 +236,7 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
|
|
|
236
236
|
return schemaMap;
|
|
237
237
|
}
|
|
238
238
|
|
|
239
|
-
// src/client/consumer/pipeline.ts
|
|
239
|
+
// src/client/kafka.client/consumer/pipeline.ts
|
|
240
240
|
function toError(error) {
|
|
241
241
|
return error instanceof Error ? error : new Error(String(error));
|
|
242
242
|
}
|
|
@@ -586,7 +586,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
586
586
|
}
|
|
587
587
|
}
|
|
588
588
|
|
|
589
|
-
// src/client/kafka.client/
|
|
589
|
+
// src/client/kafka.client/consumer/handler.ts
|
|
590
590
|
async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
591
591
|
const clockRaw = envelope.headers[HEADER_LAMPORT_CLOCK];
|
|
592
592
|
if (clockRaw === void 0) return false;
|
|
@@ -907,7 +907,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
907
907
|
);
|
|
908
908
|
}
|
|
909
909
|
|
|
910
|
-
// src/client/consumer/subscribe-retry.ts
|
|
910
|
+
// src/client/kafka.client/consumer/subscribe-retry.ts
|
|
911
911
|
async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
912
912
|
const maxAttempts = retryOpts?.retries ?? 5;
|
|
913
913
|
const backoffMs = retryOpts?.backoffMs ?? 5e3;
|
|
@@ -928,7 +928,7 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
|
928
928
|
}
|
|
929
929
|
}
|
|
930
930
|
|
|
931
|
-
// src/client/kafka.client/retry-topic.ts
|
|
931
|
+
// src/client/kafka.client/consumer/retry-topic.ts
|
|
932
932
|
async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs = 1e4) {
|
|
933
933
|
const topicSet = new Set(topics);
|
|
934
934
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -1182,9 +1182,7 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
1182
1182
|
return levelGroupIds;
|
|
1183
1183
|
}
|
|
1184
1184
|
|
|
1185
|
-
// src/client/kafka.client/
|
|
1186
|
-
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS2;
|
|
1187
|
-
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1185
|
+
// src/client/kafka.client/consumer/queue.ts
|
|
1188
1186
|
var AsyncQueue = class {
|
|
1189
1187
|
constructor(highWaterMark = Infinity, onFull = () => {
|
|
1190
1188
|
}, onDrained = () => {
|
|
@@ -1233,6 +1231,562 @@ var AsyncQueue = class {
|
|
|
1233
1231
|
return new Promise((resolve, reject) => this.waiting.push({ resolve, reject }));
|
|
1234
1232
|
}
|
|
1235
1233
|
};
|
|
1234
|
+
|
|
1235
|
+
// src/client/kafka.client/infra/circuit-breaker.ts
|
|
1236
|
+
var CircuitBreakerManager = class {
|
|
1237
|
+
constructor(deps) {
|
|
1238
|
+
this.deps = deps;
|
|
1239
|
+
}
|
|
1240
|
+
states = /* @__PURE__ */ new Map();
|
|
1241
|
+
configs = /* @__PURE__ */ new Map();
|
|
1242
|
+
setConfig(gid, options) {
|
|
1243
|
+
this.configs.set(gid, options);
|
|
1244
|
+
}
|
|
1245
|
+
/**
|
|
1246
|
+
* Returns a snapshot of the circuit breaker state for a given topic-partition.
|
|
1247
|
+
* Returns `undefined` when no state exists for the key.
|
|
1248
|
+
*/
|
|
1249
|
+
getState(topic2, partition, gid) {
|
|
1250
|
+
const state = this.states.get(`${gid}:${topic2}:${partition}`);
|
|
1251
|
+
if (!state) return void 0;
|
|
1252
|
+
return {
|
|
1253
|
+
status: state.status,
|
|
1254
|
+
failures: state.window.filter((v) => !v).length,
|
|
1255
|
+
windowSize: state.window.length
|
|
1256
|
+
};
|
|
1257
|
+
}
|
|
1258
|
+
/**
|
|
1259
|
+
* Record a failure for the given envelope and group.
|
|
1260
|
+
* Drives the CLOSED → OPEN and HALF-OPEN → OPEN transitions.
|
|
1261
|
+
*/
|
|
1262
|
+
onFailure(envelope, gid) {
|
|
1263
|
+
const cfg = this.configs.get(gid);
|
|
1264
|
+
if (!cfg) return;
|
|
1265
|
+
const threshold = cfg.threshold ?? 5;
|
|
1266
|
+
const recoveryMs = cfg.recoveryMs ?? 3e4;
|
|
1267
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
1268
|
+
let state = this.states.get(stateKey);
|
|
1269
|
+
if (!state) {
|
|
1270
|
+
state = { status: "closed", window: [], successes: 0 };
|
|
1271
|
+
this.states.set(stateKey, state);
|
|
1272
|
+
}
|
|
1273
|
+
if (state.status === "open") return;
|
|
1274
|
+
const openCircuit = () => {
|
|
1275
|
+
state.status = "open";
|
|
1276
|
+
state.window = [];
|
|
1277
|
+
state.successes = 0;
|
|
1278
|
+
clearTimeout(state.timer);
|
|
1279
|
+
for (const inst of this.deps.instrumentation)
|
|
1280
|
+
inst.onCircuitOpen?.(envelope.topic, envelope.partition);
|
|
1281
|
+
this.deps.pauseConsumer(gid, [{ topic: envelope.topic, partitions: [envelope.partition] }]);
|
|
1282
|
+
state.timer = setTimeout(() => {
|
|
1283
|
+
state.status = "half-open";
|
|
1284
|
+
state.successes = 0;
|
|
1285
|
+
this.deps.logger.log(
|
|
1286
|
+
`[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1287
|
+
);
|
|
1288
|
+
for (const inst of this.deps.instrumentation)
|
|
1289
|
+
inst.onCircuitHalfOpen?.(envelope.topic, envelope.partition);
|
|
1290
|
+
this.deps.resumeConsumer(gid, [{ topic: envelope.topic, partitions: [envelope.partition] }]);
|
|
1291
|
+
}, recoveryMs);
|
|
1292
|
+
};
|
|
1293
|
+
if (state.status === "half-open") {
|
|
1294
|
+
clearTimeout(state.timer);
|
|
1295
|
+
this.deps.logger.warn(
|
|
1296
|
+
`[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1297
|
+
);
|
|
1298
|
+
openCircuit();
|
|
1299
|
+
return;
|
|
1300
|
+
}
|
|
1301
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
1302
|
+
state.window = [...state.window, false];
|
|
1303
|
+
if (state.window.length > windowSize) {
|
|
1304
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
1305
|
+
}
|
|
1306
|
+
const failures = state.window.filter((v) => !v).length;
|
|
1307
|
+
if (failures >= threshold) {
|
|
1308
|
+
this.deps.logger.warn(
|
|
1309
|
+
`[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
|
|
1310
|
+
);
|
|
1311
|
+
openCircuit();
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
/**
|
|
1315
|
+
* Record a success for the given envelope and group.
|
|
1316
|
+
* Drives the HALF-OPEN → CLOSED transition and updates the success window.
|
|
1317
|
+
*/
|
|
1318
|
+
onSuccess(envelope, gid) {
|
|
1319
|
+
const cfg = this.configs.get(gid);
|
|
1320
|
+
if (!cfg) return;
|
|
1321
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
1322
|
+
const state = this.states.get(stateKey);
|
|
1323
|
+
if (!state) return;
|
|
1324
|
+
const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
|
|
1325
|
+
if (state.status === "half-open") {
|
|
1326
|
+
state.successes++;
|
|
1327
|
+
if (state.successes >= halfOpenSuccesses) {
|
|
1328
|
+
clearTimeout(state.timer);
|
|
1329
|
+
state.timer = void 0;
|
|
1330
|
+
state.status = "closed";
|
|
1331
|
+
state.window = [];
|
|
1332
|
+
state.successes = 0;
|
|
1333
|
+
this.deps.logger.log(
|
|
1334
|
+
`[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1335
|
+
);
|
|
1336
|
+
for (const inst of this.deps.instrumentation)
|
|
1337
|
+
inst.onCircuitClose?.(envelope.topic, envelope.partition);
|
|
1338
|
+
}
|
|
1339
|
+
} else if (state.status === "closed") {
|
|
1340
|
+
const threshold = cfg.threshold ?? 5;
|
|
1341
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
1342
|
+
state.window = [...state.window, true];
|
|
1343
|
+
if (state.window.length > windowSize) {
|
|
1344
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
/**
|
|
1349
|
+
* Remove all circuit state and config for the given group.
|
|
1350
|
+
* Called when a consumer is stopped via `stopConsumer(groupId)`.
|
|
1351
|
+
*/
|
|
1352
|
+
removeGroup(gid) {
|
|
1353
|
+
for (const key of [...this.states.keys()]) {
|
|
1354
|
+
if (key.startsWith(`${gid}:`)) {
|
|
1355
|
+
clearTimeout(this.states.get(key).timer);
|
|
1356
|
+
this.states.delete(key);
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
this.configs.delete(gid);
|
|
1360
|
+
}
|
|
1361
|
+
/** Clear all circuit state and config. Called on `disconnect()`. */
|
|
1362
|
+
clear() {
|
|
1363
|
+
for (const state of this.states.values()) clearTimeout(state.timer);
|
|
1364
|
+
this.states.clear();
|
|
1365
|
+
this.configs.clear();
|
|
1366
|
+
}
|
|
1367
|
+
};
|
|
1368
|
+
|
|
1369
|
+
// src/client/kafka.client/admin/ops.ts
|
|
1370
|
+
var AdminOps = class {
|
|
1371
|
+
constructor(deps) {
|
|
1372
|
+
this.deps = deps;
|
|
1373
|
+
}
|
|
1374
|
+
isConnected = false;
|
|
1375
|
+
/** Underlying admin client — used by index.ts for topic validation. */
|
|
1376
|
+
get admin() {
|
|
1377
|
+
return this.deps.admin;
|
|
1378
|
+
}
|
|
1379
|
+
/** Whether the admin client is currently connected. */
|
|
1380
|
+
get connected() {
|
|
1381
|
+
return this.isConnected;
|
|
1382
|
+
}
|
|
1383
|
+
/**
|
|
1384
|
+
* Connect the admin client if not already connected.
|
|
1385
|
+
* The flag is only set to `true` after a successful connect — if `admin.connect()`
|
|
1386
|
+
* throws the flag remains `false` so the next call will retry the connection.
|
|
1387
|
+
*/
|
|
1388
|
+
async ensureConnected() {
|
|
1389
|
+
if (this.isConnected) return;
|
|
1390
|
+
try {
|
|
1391
|
+
await this.deps.admin.connect();
|
|
1392
|
+
this.isConnected = true;
|
|
1393
|
+
} catch (err) {
|
|
1394
|
+
this.isConnected = false;
|
|
1395
|
+
throw err;
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
/** Disconnect admin if connected. Resets the connected flag. */
|
|
1399
|
+
async disconnect() {
|
|
1400
|
+
if (!this.isConnected) return;
|
|
1401
|
+
await this.deps.admin.disconnect();
|
|
1402
|
+
this.isConnected = false;
|
|
1403
|
+
}
|
|
1404
|
+
async resetOffsets(groupId, topic2, position) {
|
|
1405
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1406
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1407
|
+
throw new Error(
|
|
1408
|
+
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
1409
|
+
);
|
|
1410
|
+
}
|
|
1411
|
+
await this.ensureConnected();
|
|
1412
|
+
const partitionOffsets = await this.deps.admin.fetchTopicOffsets(topic2);
|
|
1413
|
+
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
1414
|
+
partition,
|
|
1415
|
+
offset: position === "earliest" ? low : high
|
|
1416
|
+
}));
|
|
1417
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1418
|
+
this.deps.logger.log(
|
|
1419
|
+
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1420
|
+
);
|
|
1421
|
+
}
|
|
1422
|
+
/**
|
|
1423
|
+
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
1424
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
1425
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1426
|
+
*/
|
|
1427
|
+
async seekToOffset(groupId, assignments) {
|
|
1428
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1429
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1430
|
+
throw new Error(
|
|
1431
|
+
`seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1434
|
+
await this.ensureConnected();
|
|
1435
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
1436
|
+
for (const { topic: topic2, partition, offset } of assignments) {
|
|
1437
|
+
const list = byTopic.get(topic2) ?? [];
|
|
1438
|
+
list.push({ partition, offset });
|
|
1439
|
+
byTopic.set(topic2, list);
|
|
1440
|
+
}
|
|
1441
|
+
for (const [topic2, partitions] of byTopic) {
|
|
1442
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1443
|
+
this.deps.logger.log(
|
|
1444
|
+
`Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
|
|
1445
|
+
);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
/**
|
|
1449
|
+
* Seek specific topic-partition pairs to the offset nearest to a given timestamp
|
|
1450
|
+
* (in milliseconds) for a stopped consumer group.
|
|
1451
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
1452
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1453
|
+
* If no offset exists at the requested timestamp (e.g. empty partition or
|
|
1454
|
+
* future timestamp), the partition falls back to `-1` (end of topic — new messages only).
|
|
1455
|
+
*/
|
|
1456
|
+
async seekToTimestamp(groupId, assignments) {
|
|
1457
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1458
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1459
|
+
throw new Error(
|
|
1460
|
+
`seekToTimestamp: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1461
|
+
);
|
|
1462
|
+
}
|
|
1463
|
+
await this.ensureConnected();
|
|
1464
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
1465
|
+
for (const { topic: topic2, partition, timestamp } of assignments) {
|
|
1466
|
+
const list = byTopic.get(topic2) ?? [];
|
|
1467
|
+
list.push({ partition, timestamp });
|
|
1468
|
+
byTopic.set(topic2, list);
|
|
1469
|
+
}
|
|
1470
|
+
for (const [topic2, parts] of byTopic) {
|
|
1471
|
+
const offsets = await Promise.all(
|
|
1472
|
+
parts.map(async ({ partition, timestamp }) => {
|
|
1473
|
+
const results = await this.deps.admin.fetchTopicOffsetsByTime(
|
|
1474
|
+
topic2,
|
|
1475
|
+
timestamp
|
|
1476
|
+
);
|
|
1477
|
+
const found = results.find(
|
|
1478
|
+
(r) => r.partition === partition
|
|
1479
|
+
);
|
|
1480
|
+
return { partition, offset: found?.offset ?? "-1" };
|
|
1481
|
+
})
|
|
1482
|
+
);
|
|
1483
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
|
|
1484
|
+
this.deps.logger.log(
|
|
1485
|
+
`Offsets set by timestamp for group "${gid}" on "${topic2}": ${JSON.stringify(offsets)}`
|
|
1486
|
+
);
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
/**
|
|
1490
|
+
* Query consumer group lag per partition.
|
|
1491
|
+
* Lag = broker high-watermark − last committed offset.
|
|
1492
|
+
* A committed offset of -1 (nothing committed yet) counts as full lag.
|
|
1493
|
+
*
|
|
1494
|
+
* Returns an empty array when the consumer group has never committed any
|
|
1495
|
+
* offsets (freshly created group, `autoCommit: false` with no manual commits,
|
|
1496
|
+
* or group not yet assigned). This is a Kafka protocol limitation:
|
|
1497
|
+
* `fetchOffsets` only returns data for topic-partitions that have at least one
|
|
1498
|
+
* committed offset. Use `checkStatus()` to verify broker connectivity in that case.
|
|
1499
|
+
*/
|
|
1500
|
+
async getConsumerLag(groupId) {
|
|
1501
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1502
|
+
await this.ensureConnected();
|
|
1503
|
+
const committedByTopic = await this.deps.admin.fetchOffsets({ groupId: gid });
|
|
1504
|
+
const brokerOffsetsAll = await Promise.all(
|
|
1505
|
+
committedByTopic.map(({ topic: topic2 }) => this.deps.admin.fetchTopicOffsets(topic2))
|
|
1506
|
+
);
|
|
1507
|
+
const result = [];
|
|
1508
|
+
for (let i = 0; i < committedByTopic.length; i++) {
|
|
1509
|
+
const { topic: topic2, partitions } = committedByTopic[i];
|
|
1510
|
+
const brokerOffsets = brokerOffsetsAll[i];
|
|
1511
|
+
for (const { partition, offset } of partitions) {
|
|
1512
|
+
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
1513
|
+
if (!broker) continue;
|
|
1514
|
+
const committed = parseInt(offset, 10);
|
|
1515
|
+
const high = parseInt(broker.high, 10);
|
|
1516
|
+
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
1517
|
+
result.push({ topic: topic2, partition, lag });
|
|
1518
|
+
}
|
|
1519
|
+
}
|
|
1520
|
+
return result;
|
|
1521
|
+
}
|
|
1522
|
+
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
1523
|
+
async checkStatus() {
|
|
1524
|
+
try {
|
|
1525
|
+
await this.ensureConnected();
|
|
1526
|
+
const topics = await this.deps.admin.listTopics();
|
|
1527
|
+
return { status: "up", clientId: this.deps.clientId, topics };
|
|
1528
|
+
} catch (error) {
|
|
1529
|
+
return {
|
|
1530
|
+
status: "down",
|
|
1531
|
+
clientId: this.deps.clientId,
|
|
1532
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1533
|
+
};
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
/**
|
|
1537
|
+
* List all consumer groups known to the broker.
|
|
1538
|
+
* Useful for monitoring which groups are active and their current state.
|
|
1539
|
+
*/
|
|
1540
|
+
async listConsumerGroups() {
|
|
1541
|
+
await this.ensureConnected();
|
|
1542
|
+
const result = await this.deps.admin.listGroups();
|
|
1543
|
+
return result.groups.map((g) => ({
|
|
1544
|
+
groupId: g.groupId,
|
|
1545
|
+
state: g.state ?? "Unknown"
|
|
1546
|
+
}));
|
|
1547
|
+
}
|
|
1548
|
+
/**
|
|
1549
|
+
* Describe topics — returns partition layout, leader, replicas, and ISR.
|
|
1550
|
+
* @param topics Topic names to describe. Omit to describe all topics.
|
|
1551
|
+
*/
|
|
1552
|
+
async describeTopics(topics) {
|
|
1553
|
+
await this.ensureConnected();
|
|
1554
|
+
const result = await this.deps.admin.fetchTopicMetadata(
|
|
1555
|
+
topics ? { topics } : void 0
|
|
1556
|
+
);
|
|
1557
|
+
return result.topics.map((t) => ({
|
|
1558
|
+
name: t.name,
|
|
1559
|
+
partitions: t.partitions.map((p) => ({
|
|
1560
|
+
partition: p.partitionId ?? p.partition,
|
|
1561
|
+
leader: p.leader,
|
|
1562
|
+
replicas: p.replicas.map(
|
|
1563
|
+
(r) => typeof r === "number" ? r : r.nodeId
|
|
1564
|
+
),
|
|
1565
|
+
isr: p.isr.map(
|
|
1566
|
+
(r) => typeof r === "number" ? r : r.nodeId
|
|
1567
|
+
)
|
|
1568
|
+
}))
|
|
1569
|
+
}));
|
|
1570
|
+
}
|
|
1571
|
+
/**
|
|
1572
|
+
* Delete records from a topic up to (but not including) the given offsets.
|
|
1573
|
+
* All messages with offsets **before** the given offset are deleted.
|
|
1574
|
+
*/
|
|
1575
|
+
async deleteRecords(topic2, partitions) {
|
|
1576
|
+
await this.ensureConnected();
|
|
1577
|
+
await this.deps.admin.deleteTopicRecords({ topic: topic2, partitions });
|
|
1578
|
+
}
|
|
1579
|
+
/**
|
|
1580
|
+
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
1581
|
+
* `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
|
|
1582
|
+
* rather than silently discovering missing topics on the first handler failure.
|
|
1583
|
+
*/
|
|
1584
|
+
async validateRetryTopicsExist(topicNames, maxRetries) {
|
|
1585
|
+
await this.ensureConnected();
|
|
1586
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1587
|
+
const missing = [];
|
|
1588
|
+
for (const t of topicNames) {
|
|
1589
|
+
for (let level = 1; level <= maxRetries; level++) {
|
|
1590
|
+
const retryTopic = `${t}.retry.${level}`;
|
|
1591
|
+
if (!existing.has(retryTopic)) missing.push(retryTopic);
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
if (missing.length > 0) {
|
|
1595
|
+
throw new Error(
|
|
1596
|
+
`retryTopics: true but the following retry topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1597
|
+
);
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
/**
|
|
1601
|
+
* When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
|
|
1602
|
+
* consumed topic. Throws a clear error at startup rather than silently discovering
|
|
1603
|
+
* missing DLQ topics on the first handler failure.
|
|
1604
|
+
*/
|
|
1605
|
+
async validateDlqTopicsExist(topicNames) {
|
|
1606
|
+
await this.ensureConnected();
|
|
1607
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1608
|
+
const missing = topicNames.filter((t) => !existing.has(`${t}.dlq`)).map((t) => `${t}.dlq`);
|
|
1609
|
+
if (missing.length > 0) {
|
|
1610
|
+
throw new Error(
|
|
1611
|
+
`dlq: true but the following DLQ topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1612
|
+
);
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
/**
|
|
1616
|
+
* When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
|
|
1617
|
+
* that every `<topic>.duplicates` destination topic already exists. Throws a
|
|
1618
|
+
* clear error at startup rather than silently dropping duplicates on first hit.
|
|
1619
|
+
*/
|
|
1620
|
+
async validateDuplicatesTopicsExist(topicNames, customDestination) {
|
|
1621
|
+
await this.ensureConnected();
|
|
1622
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1623
|
+
const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
|
|
1624
|
+
const missing = toCheck.filter((t) => !existing.has(t));
|
|
1625
|
+
if (missing.length > 0) {
|
|
1626
|
+
throw new Error(
|
|
1627
|
+
`deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1628
|
+
);
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
};
|
|
1632
|
+
|
|
1633
|
+
// src/client/kafka.client/consumer/dlq-replay.ts
|
|
1634
|
+
async function replayDlqTopic(topic2, options = {}, deps) {
|
|
1635
|
+
const dlqTopic = `${topic2}.dlq`;
|
|
1636
|
+
const partitionOffsets = await deps.fetchTopicOffsets(dlqTopic);
|
|
1637
|
+
const activePartitions = partitionOffsets.filter((p) => parseInt(p.high, 10) > 0);
|
|
1638
|
+
if (activePartitions.length === 0) {
|
|
1639
|
+
deps.logger.log(`replayDlq: "${dlqTopic}" is empty \u2014 nothing to replay`);
|
|
1640
|
+
return { replayed: 0, skipped: 0 };
|
|
1641
|
+
}
|
|
1642
|
+
const highWatermarks = new Map(
|
|
1643
|
+
activePartitions.map(({ partition, high }) => [partition, parseInt(high, 10)])
|
|
1644
|
+
);
|
|
1645
|
+
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1646
|
+
let replayed = 0;
|
|
1647
|
+
let skipped = 0;
|
|
1648
|
+
const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
|
|
1649
|
+
await new Promise((resolve, reject) => {
|
|
1650
|
+
const consumer = deps.createConsumer(tempGroupId);
|
|
1651
|
+
const cleanup = () => deps.cleanupConsumer(consumer, tempGroupId);
|
|
1652
|
+
consumer.connect().then(() => subscribeWithRetry(consumer, [dlqTopic], deps.logger)).then(
|
|
1653
|
+
() => consumer.run({
|
|
1654
|
+
eachMessage: async ({ partition, message }) => {
|
|
1655
|
+
if (!message.value) return;
|
|
1656
|
+
const offset = parseInt(message.offset, 10);
|
|
1657
|
+
processedOffsets.set(partition, offset);
|
|
1658
|
+
const headers = decodeHeaders(message.headers);
|
|
1659
|
+
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1660
|
+
const originalHeaders = Object.fromEntries(
|
|
1661
|
+
Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
|
|
1662
|
+
);
|
|
1663
|
+
const value = message.value.toString();
|
|
1664
|
+
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1665
|
+
if (!targetTopic || !shouldProcess) {
|
|
1666
|
+
skipped++;
|
|
1667
|
+
} else if (options.dryRun) {
|
|
1668
|
+
deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
|
|
1669
|
+
replayed++;
|
|
1670
|
+
} else {
|
|
1671
|
+
await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
|
|
1672
|
+
replayed++;
|
|
1673
|
+
}
|
|
1674
|
+
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1675
|
+
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1676
|
+
);
|
|
1677
|
+
if (allDone) {
|
|
1678
|
+
cleanup();
|
|
1679
|
+
resolve();
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
})
|
|
1683
|
+
).catch((err) => {
|
|
1684
|
+
cleanup();
|
|
1685
|
+
reject(err);
|
|
1686
|
+
});
|
|
1687
|
+
});
|
|
1688
|
+
deps.logger.log(`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`);
|
|
1689
|
+
return { replayed, skipped };
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
// src/client/kafka.client/infra/metrics-manager.ts
|
|
1693
|
+
var MetricsManager = class {
|
|
1694
|
+
constructor(deps) {
|
|
1695
|
+
this.deps = deps;
|
|
1696
|
+
}
|
|
1697
|
+
topicMetrics = /* @__PURE__ */ new Map();
|
|
1698
|
+
metricsFor(topic2) {
|
|
1699
|
+
let m = this.topicMetrics.get(topic2);
|
|
1700
|
+
if (!m) {
|
|
1701
|
+
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1702
|
+
this.topicMetrics.set(topic2, m);
|
|
1703
|
+
}
|
|
1704
|
+
return m;
|
|
1705
|
+
}
|
|
1706
|
+
/** Fire `afterSend` instrumentation hooks for each message in a batch. */
|
|
1707
|
+
notifyAfterSend(topic2, count) {
|
|
1708
|
+
for (let i = 0; i < count; i++)
|
|
1709
|
+
for (const inst of this.deps.instrumentation) inst.afterSend?.(topic2);
|
|
1710
|
+
}
|
|
1711
|
+
notifyRetry(envelope, attempt, maxRetries) {
|
|
1712
|
+
this.metricsFor(envelope.topic).retryCount++;
|
|
1713
|
+
for (const inst of this.deps.instrumentation) inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1714
|
+
}
|
|
1715
|
+
notifyDlq(envelope, reason, gid) {
|
|
1716
|
+
this.metricsFor(envelope.topic).dlqCount++;
|
|
1717
|
+
for (const inst of this.deps.instrumentation) inst.onDlq?.(envelope, reason);
|
|
1718
|
+
if (gid) this.deps.onCircuitFailure(envelope, gid);
|
|
1719
|
+
}
|
|
1720
|
+
notifyDuplicate(envelope, strategy) {
|
|
1721
|
+
this.metricsFor(envelope.topic).dedupCount++;
|
|
1722
|
+
for (const inst of this.deps.instrumentation) inst.onDuplicate?.(envelope, strategy);
|
|
1723
|
+
}
|
|
1724
|
+
notifyMessage(envelope, gid) {
|
|
1725
|
+
this.metricsFor(envelope.topic).processedCount++;
|
|
1726
|
+
for (const inst of this.deps.instrumentation) inst.onMessage?.(envelope);
|
|
1727
|
+
if (gid) this.deps.onCircuitSuccess(envelope, gid);
|
|
1728
|
+
}
|
|
1729
|
+
getMetrics(topic2) {
|
|
1730
|
+
if (topic2 !== void 0) {
|
|
1731
|
+
const m = this.topicMetrics.get(topic2);
|
|
1732
|
+
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1733
|
+
}
|
|
1734
|
+
const agg = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1735
|
+
for (const m of this.topicMetrics.values()) {
|
|
1736
|
+
agg.processedCount += m.processedCount;
|
|
1737
|
+
agg.retryCount += m.retryCount;
|
|
1738
|
+
agg.dlqCount += m.dlqCount;
|
|
1739
|
+
agg.dedupCount += m.dedupCount;
|
|
1740
|
+
}
|
|
1741
|
+
return agg;
|
|
1742
|
+
}
|
|
1743
|
+
resetMetrics(topic2) {
|
|
1744
|
+
if (topic2 !== void 0) {
|
|
1745
|
+
this.topicMetrics.delete(topic2);
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1748
|
+
this.topicMetrics.clear();
|
|
1749
|
+
}
|
|
1750
|
+
};
|
|
1751
|
+
|
|
1752
|
+
// src/client/kafka.client/infra/inflight-tracker.ts
|
|
1753
|
+
var InFlightTracker = class {
|
|
1754
|
+
constructor(warn) {
|
|
1755
|
+
this.warn = warn;
|
|
1756
|
+
}
|
|
1757
|
+
inFlightTotal = 0;
|
|
1758
|
+
drainResolvers = [];
|
|
1759
|
+
track(fn) {
|
|
1760
|
+
this.inFlightTotal++;
|
|
1761
|
+
return fn().finally(() => {
|
|
1762
|
+
this.inFlightTotal--;
|
|
1763
|
+
if (this.inFlightTotal === 0) this.drainResolvers.splice(0).forEach((r) => r());
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1766
|
+
waitForDrain(timeoutMs) {
|
|
1767
|
+
if (this.inFlightTotal === 0) return Promise.resolve();
|
|
1768
|
+
return new Promise((resolve) => {
|
|
1769
|
+
let handle;
|
|
1770
|
+
const onDrain = () => {
|
|
1771
|
+
clearTimeout(handle);
|
|
1772
|
+
resolve();
|
|
1773
|
+
};
|
|
1774
|
+
this.drainResolvers.push(onDrain);
|
|
1775
|
+
handle = setTimeout(() => {
|
|
1776
|
+
const idx = this.drainResolvers.indexOf(onDrain);
|
|
1777
|
+
if (idx !== -1) this.drainResolvers.splice(idx, 1);
|
|
1778
|
+
this.warn(
|
|
1779
|
+
`Drain timed out after ${timeoutMs}ms \u2014 ${this.inFlightTotal} handler(s) still in flight`
|
|
1780
|
+
);
|
|
1781
|
+
resolve();
|
|
1782
|
+
}, timeoutMs);
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
};
|
|
1786
|
+
|
|
1787
|
+
// src/client/kafka.client/index.ts
|
|
1788
|
+
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS2;
|
|
1789
|
+
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1236
1790
|
var KafkaClient = class _KafkaClient {
|
|
1237
1791
|
kafka;
|
|
1238
1792
|
producer;
|
|
@@ -1241,7 +1795,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1241
1795
|
/** Maps transactionalId → Producer for each active retry level consumer. */
|
|
1242
1796
|
retryTxProducers = /* @__PURE__ */ new Map();
|
|
1243
1797
|
consumers = /* @__PURE__ */ new Map();
|
|
1244
|
-
admin;
|
|
1245
1798
|
logger;
|
|
1246
1799
|
autoCreateTopicsEnabled;
|
|
1247
1800
|
strictSchemasEnabled;
|
|
@@ -1261,20 +1814,28 @@ var KafkaClient = class _KafkaClient {
|
|
|
1261
1814
|
onRebalance;
|
|
1262
1815
|
/** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
|
|
1263
1816
|
txId;
|
|
1264
|
-
/** Per-topic event counters, lazily created on first event. Aggregated by `getMetrics()`. */
|
|
1265
|
-
_topicMetrics = /* @__PURE__ */ new Map();
|
|
1266
1817
|
/** Monotonically increasing Lamport clock stamped on every outgoing message. */
|
|
1267
1818
|
_lamportClock = 0;
|
|
1819
|
+
/** Topics to scan for the highest Lamport clock value on `connectProducer()`. */
|
|
1820
|
+
clockRecoveryTopics;
|
|
1268
1821
|
/** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
|
|
1269
1822
|
dedupStates = /* @__PURE__ */ new Map();
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
isAdminConnected = false;
|
|
1275
|
-
inFlightTotal = 0;
|
|
1276
|
-
drainResolvers = [];
|
|
1823
|
+
circuitBreaker;
|
|
1824
|
+
adminOps;
|
|
1825
|
+
metrics;
|
|
1826
|
+
inFlight;
|
|
1277
1827
|
clientId;
|
|
1828
|
+
_producerOpsDeps;
|
|
1829
|
+
_consumerOpsDeps;
|
|
1830
|
+
_retryTopicDeps;
|
|
1831
|
+
/** DLQ header keys added by the pipeline — stripped before re-publishing. */
|
|
1832
|
+
static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
1833
|
+
"x-dlq-original-topic",
|
|
1834
|
+
"x-dlq-failed-at",
|
|
1835
|
+
"x-dlq-error-message",
|
|
1836
|
+
"x-dlq-error-stack",
|
|
1837
|
+
"x-dlq-attempt-count"
|
|
1838
|
+
]);
|
|
1278
1839
|
constructor(clientId, groupId, brokers, options) {
|
|
1279
1840
|
this.clientId = clientId;
|
|
1280
1841
|
this.defaultGroupId = groupId;
|
|
@@ -1292,6 +1853,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1292
1853
|
this.onTtlExpired = options?.onTtlExpired;
|
|
1293
1854
|
this.onRebalance = options?.onRebalance;
|
|
1294
1855
|
this.txId = options?.transactionalId ?? `${clientId}-tx`;
|
|
1856
|
+
this.clockRecoveryTopics = options?.clockRecovery?.topics ?? [];
|
|
1295
1857
|
this.kafka = new KafkaClass({
|
|
1296
1858
|
kafkaJS: {
|
|
1297
1859
|
clientId: this.clientId,
|
|
@@ -1299,12 +1861,41 @@ var KafkaClient = class _KafkaClient {
|
|
|
1299
1861
|
logLevel: KafkaLogLevel.ERROR
|
|
1300
1862
|
}
|
|
1301
1863
|
});
|
|
1302
|
-
this.producer = this.kafka.producer({
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1864
|
+
this.producer = this.kafka.producer({ kafkaJS: { acks: -1 } });
|
|
1865
|
+
this.adminOps = new AdminOps({
|
|
1866
|
+
admin: this.kafka.admin(),
|
|
1867
|
+
logger: this.logger,
|
|
1868
|
+
runningConsumers: this.runningConsumers,
|
|
1869
|
+
defaultGroupId: this.defaultGroupId,
|
|
1870
|
+
clientId: this.clientId
|
|
1871
|
+
});
|
|
1872
|
+
this.circuitBreaker = new CircuitBreakerManager({
|
|
1873
|
+
pauseConsumer: (gid, assignments) => this.pauseConsumer(gid, assignments),
|
|
1874
|
+
resumeConsumer: (gid, assignments) => this.resumeConsumer(gid, assignments),
|
|
1875
|
+
logger: this.logger,
|
|
1876
|
+
instrumentation: this.instrumentation
|
|
1877
|
+
});
|
|
1878
|
+
this.metrics = new MetricsManager({
|
|
1879
|
+
instrumentation: this.instrumentation,
|
|
1880
|
+
onCircuitFailure: (envelope, gid) => this.circuitBreaker.onFailure(envelope, gid),
|
|
1881
|
+
onCircuitSuccess: (envelope, gid) => this.circuitBreaker.onSuccess(envelope, gid)
|
|
1306
1882
|
});
|
|
1307
|
-
this.
|
|
1883
|
+
this.inFlight = new InFlightTracker((msg) => this.logger.warn(msg));
|
|
1884
|
+
this._producerOpsDeps = {
|
|
1885
|
+
schemaRegistry: this.schemaRegistry,
|
|
1886
|
+
strictSchemasEnabled: this.strictSchemasEnabled,
|
|
1887
|
+
instrumentation: this.instrumentation,
|
|
1888
|
+
logger: this.logger,
|
|
1889
|
+
nextLamportClock: () => ++this._lamportClock
|
|
1890
|
+
};
|
|
1891
|
+
this._consumerOpsDeps = {
|
|
1892
|
+
consumers: this.consumers,
|
|
1893
|
+
consumerCreationOptions: this.consumerCreationOptions,
|
|
1894
|
+
kafka: this.kafka,
|
|
1895
|
+
onRebalance: this.onRebalance,
|
|
1896
|
+
logger: this.logger
|
|
1897
|
+
};
|
|
1898
|
+
this._retryTopicDeps = this.buildRetryTopicDeps();
|
|
1308
1899
|
}
|
|
1309
1900
|
async sendMessage(topicOrDesc, message, options = {}) {
|
|
1310
1901
|
const payload = await this.preparePayload(
|
|
@@ -1322,7 +1913,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1322
1913
|
options.compression
|
|
1323
1914
|
);
|
|
1324
1915
|
await this.producer.send(payload);
|
|
1325
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1916
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1326
1917
|
}
|
|
1327
1918
|
/**
|
|
1328
1919
|
* Send a null-value (tombstone) message. Used with log-compacted topics to signal
|
|
@@ -1336,27 +1927,16 @@ var KafkaClient = class _KafkaClient {
|
|
|
1336
1927
|
* @param headers Optional custom Kafka headers.
|
|
1337
1928
|
*/
|
|
1338
1929
|
async sendTombstone(topic2, key, headers) {
|
|
1339
|
-
const hdrs = { ...headers };
|
|
1340
|
-
for (const inst of this.instrumentation)
|
|
1341
|
-
|
|
1342
|
-
}
|
|
1343
|
-
|
|
1344
|
-
await this.producer.send({
|
|
1345
|
-
topic: topic2,
|
|
1346
|
-
messages: [{ value: null, key, headers: hdrs }]
|
|
1347
|
-
});
|
|
1348
|
-
for (const inst of this.instrumentation) {
|
|
1349
|
-
inst.afterSend?.(topic2);
|
|
1350
|
-
}
|
|
1930
|
+
const hdrs = { ...headers };
|
|
1931
|
+
for (const inst of this.instrumentation) inst.beforeSend?.(topic2, hdrs);
|
|
1932
|
+
await this.ensureTopic(topic2);
|
|
1933
|
+
await this.producer.send({ topic: topic2, messages: [{ value: null, key, headers: hdrs }] });
|
|
1934
|
+
for (const inst of this.instrumentation) inst.afterSend?.(topic2);
|
|
1351
1935
|
}
|
|
1352
1936
|
async sendBatch(topicOrDesc, messages, options) {
|
|
1353
|
-
const payload = await this.preparePayload(
|
|
1354
|
-
topicOrDesc,
|
|
1355
|
-
messages,
|
|
1356
|
-
options?.compression
|
|
1357
|
-
);
|
|
1937
|
+
const payload = await this.preparePayload(topicOrDesc, messages, options?.compression);
|
|
1358
1938
|
await this.producer.send(payload);
|
|
1359
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1939
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1360
1940
|
}
|
|
1361
1941
|
/** Execute multiple sends atomically. Commits on success, aborts on error. */
|
|
1362
1942
|
async transaction(fn) {
|
|
@@ -1368,12 +1948,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1368
1948
|
}
|
|
1369
1949
|
const initPromise = (async () => {
|
|
1370
1950
|
const p = this.kafka.producer({
|
|
1371
|
-
kafkaJS: {
|
|
1372
|
-
acks: -1,
|
|
1373
|
-
idempotent: true,
|
|
1374
|
-
transactionalId: this.txId,
|
|
1375
|
-
maxInFlightRequests: 1
|
|
1376
|
-
}
|
|
1951
|
+
kafkaJS: { acks: -1, idempotent: true, transactionalId: this.txId, maxInFlightRequests: 1 }
|
|
1377
1952
|
});
|
|
1378
1953
|
await p.connect();
|
|
1379
1954
|
_activeTransactionalIds.add(this.txId);
|
|
@@ -1400,19 +1975,12 @@ var KafkaClient = class _KafkaClient {
|
|
|
1400
1975
|
}
|
|
1401
1976
|
]);
|
|
1402
1977
|
await tx.send(payload);
|
|
1403
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1978
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1404
1979
|
},
|
|
1405
|
-
/**
|
|
1406
|
-
* Send multiple messages in a single call to the topic.
|
|
1407
|
-
* All messages in the batch will be sent atomically.
|
|
1408
|
-
* If any message fails to send, the entire batch will be aborted.
|
|
1409
|
-
* @param topicOrDesc - topic name or TopicDescriptor
|
|
1410
|
-
* @param messages - array of messages to send with optional key, headers, correlationId, schemaVersion, and eventId
|
|
1411
|
-
*/
|
|
1412
1980
|
sendBatch: async (topicOrDesc, messages) => {
|
|
1413
1981
|
const payload = await this.preparePayload(topicOrDesc, messages);
|
|
1414
1982
|
await tx.send(payload);
|
|
1415
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1983
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1416
1984
|
}
|
|
1417
1985
|
};
|
|
1418
1986
|
await fn(ctx);
|
|
@@ -1421,10 +1989,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1421
1989
|
try {
|
|
1422
1990
|
await tx.abort();
|
|
1423
1991
|
} catch (abortError) {
|
|
1424
|
-
this.logger.error(
|
|
1425
|
-
"Failed to abort transaction:",
|
|
1426
|
-
toError(abortError).message
|
|
1427
|
-
);
|
|
1992
|
+
this.logger.error("Failed to abort transaction:", toError(abortError).message);
|
|
1428
1993
|
}
|
|
1429
1994
|
throw error;
|
|
1430
1995
|
}
|
|
@@ -1436,8 +2001,109 @@ var KafkaClient = class _KafkaClient {
|
|
|
1436
2001
|
*/
|
|
1437
2002
|
async connectProducer() {
|
|
1438
2003
|
await this.producer.connect();
|
|
2004
|
+
await this.recoverLamportClock(this.clockRecoveryTopics);
|
|
1439
2005
|
this.logger.log("Producer connected");
|
|
1440
2006
|
}
|
|
2007
|
+
/**
|
|
2008
|
+
* Recover the Lamport clock from the last message across the given topics.
|
|
2009
|
+
*
|
|
2010
|
+
* For each topic, fetches partition high-watermarks via admin, creates a
|
|
2011
|
+
* short-lived consumer, seeks every non-empty partition to its last offset
|
|
2012
|
+
* (`highWatermark − 1`), reads one message per partition, and extracts the
|
|
2013
|
+
* maximum `x-lamport-clock` header value. On completion `_lamportClock` is
|
|
2014
|
+
* set to that maximum so the next `++_lamportClock` yields a strictly greater
|
|
2015
|
+
* value than any previously sent clock.
|
|
2016
|
+
*
|
|
2017
|
+
* Topics that are empty or missing are silently skipped.
|
|
2018
|
+
*/
|
|
2019
|
+
async recoverLamportClock(topics) {
|
|
2020
|
+
if (topics.length === 0) return;
|
|
2021
|
+
this.logger.log(
|
|
2022
|
+
`Clock recovery: scanning ${topics.length} topic(s) for Lamport clock...`
|
|
2023
|
+
);
|
|
2024
|
+
await this.adminOps.ensureConnected();
|
|
2025
|
+
const partitionsToRead = [];
|
|
2026
|
+
for (const t of topics) {
|
|
2027
|
+
let offsets;
|
|
2028
|
+
try {
|
|
2029
|
+
offsets = await this.adminOps.admin.fetchTopicOffsets(t);
|
|
2030
|
+
} catch {
|
|
2031
|
+
this.logger.warn(
|
|
2032
|
+
`Clock recovery: could not fetch offsets for "${t}", skipping`
|
|
2033
|
+
);
|
|
2034
|
+
continue;
|
|
2035
|
+
}
|
|
2036
|
+
for (const { partition, high, low } of offsets) {
|
|
2037
|
+
if (parseInt(high, 10) > parseInt(low, 10)) {
|
|
2038
|
+
partitionsToRead.push({
|
|
2039
|
+
topic: t,
|
|
2040
|
+
partition,
|
|
2041
|
+
lastOffset: String(parseInt(high, 10) - 1)
|
|
2042
|
+
});
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
if (partitionsToRead.length === 0) {
|
|
2047
|
+
this.logger.log(
|
|
2048
|
+
"Clock recovery: all topics empty \u2014 keeping Lamport clock at 0"
|
|
2049
|
+
);
|
|
2050
|
+
return;
|
|
2051
|
+
}
|
|
2052
|
+
const recoveryGroupId = `${this.clientId}-clock-recovery-${Date.now()}`;
|
|
2053
|
+
let maxClock = -1;
|
|
2054
|
+
await new Promise((resolve, reject) => {
|
|
2055
|
+
const consumer = this.kafka.consumer({
|
|
2056
|
+
kafkaJS: { groupId: recoveryGroupId }
|
|
2057
|
+
});
|
|
2058
|
+
const remaining = new Set(
|
|
2059
|
+
partitionsToRead.map((p) => `${p.topic}:${p.partition}`)
|
|
2060
|
+
);
|
|
2061
|
+
const cleanup = () => {
|
|
2062
|
+
consumer.disconnect().catch(() => {
|
|
2063
|
+
});
|
|
2064
|
+
};
|
|
2065
|
+
consumer.connect().then(async () => {
|
|
2066
|
+
const uniqueTopics = [
|
|
2067
|
+
...new Set(partitionsToRead.map((p) => p.topic))
|
|
2068
|
+
];
|
|
2069
|
+
await consumer.subscribe({ topics: uniqueTopics });
|
|
2070
|
+
for (const { topic: t, partition, lastOffset } of partitionsToRead) {
|
|
2071
|
+
consumer.seek({ topic: t, partition, offset: lastOffset });
|
|
2072
|
+
}
|
|
2073
|
+
}).then(
|
|
2074
|
+
() => consumer.run({
|
|
2075
|
+
eachMessage: async ({ topic: t, partition, message }) => {
|
|
2076
|
+
const key = `${t}:${partition}`;
|
|
2077
|
+
if (!remaining.has(key)) return;
|
|
2078
|
+
remaining.delete(key);
|
|
2079
|
+
const clockHeader = message.headers?.[HEADER_LAMPORT_CLOCK];
|
|
2080
|
+
if (clockHeader !== void 0) {
|
|
2081
|
+
const raw = Buffer.isBuffer(clockHeader) ? clockHeader.toString() : String(clockHeader);
|
|
2082
|
+
const clock = Number(raw);
|
|
2083
|
+
if (!isNaN(clock) && clock > maxClock) maxClock = clock;
|
|
2084
|
+
}
|
|
2085
|
+
if (remaining.size === 0) {
|
|
2086
|
+
cleanup();
|
|
2087
|
+
resolve();
|
|
2088
|
+
}
|
|
2089
|
+
}
|
|
2090
|
+
})
|
|
2091
|
+
).catch((err) => {
|
|
2092
|
+
cleanup();
|
|
2093
|
+
reject(err);
|
|
2094
|
+
});
|
|
2095
|
+
});
|
|
2096
|
+
if (maxClock >= 0) {
|
|
2097
|
+
this._lamportClock = maxClock;
|
|
2098
|
+
this.logger.log(
|
|
2099
|
+
`Clock recovery: Lamport clock restored \u2014 next clock will be ${maxClock + 1}`
|
|
2100
|
+
);
|
|
2101
|
+
} else {
|
|
2102
|
+
this.logger.log(
|
|
2103
|
+
"Clock recovery: no x-lamport-clock headers found \u2014 keeping clock at 0"
|
|
2104
|
+
);
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
1441
2107
|
/**
|
|
1442
2108
|
* @internal Not part of `IKafkaClient` — use `disconnect()` for full teardown.
|
|
1443
2109
|
*/
|
|
@@ -1446,35 +2112,14 @@ var KafkaClient = class _KafkaClient {
|
|
|
1446
2112
|
this.logger.log("Producer disconnected");
|
|
1447
2113
|
}
|
|
1448
2114
|
async startConsumer(topics, handleMessage, options = {}) {
|
|
1449
|
-
|
|
1450
|
-
throw new Error(
|
|
1451
|
-
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1452
|
-
);
|
|
1453
|
-
}
|
|
1454
|
-
const hasRegexTopics = topics.some((t) => t instanceof RegExp);
|
|
1455
|
-
if (options.retryTopics && hasRegexTopics) {
|
|
1456
|
-
throw new Error(
|
|
1457
|
-
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
1458
|
-
);
|
|
1459
|
-
}
|
|
2115
|
+
this.validateTopicConsumerOpts(topics, options);
|
|
1460
2116
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1461
2117
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", setupOptions);
|
|
1462
|
-
if (options.circuitBreaker)
|
|
1463
|
-
this.circuitConfigs.set(gid, options.circuitBreaker);
|
|
2118
|
+
if (options.circuitBreaker) this.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
1464
2119
|
const deps = this.messageDepsFor(gid);
|
|
1465
|
-
const
|
|
1466
|
-
const deduplication = this.resolveDeduplicationContext(
|
|
1467
|
-
gid,
|
|
1468
|
-
options.deduplication
|
|
1469
|
-
);
|
|
1470
|
-
let eosMainContext;
|
|
1471
|
-
if (options.retryTopics && retry) {
|
|
1472
|
-
const mainTxId = `${gid}-main-tx`;
|
|
1473
|
-
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1474
|
-
eosMainContext = { txProducer, consumer };
|
|
1475
|
-
}
|
|
2120
|
+
const eosMainContext = await this.makeEosMainContext(gid, consumer, options);
|
|
1476
2121
|
await consumer.run({
|
|
1477
|
-
eachMessage: (payload) => this.
|
|
2122
|
+
eachMessage: (payload) => this.inFlight.track(
|
|
1478
2123
|
() => handleEachMessage(
|
|
1479
2124
|
payload,
|
|
1480
2125
|
{
|
|
@@ -1484,9 +2129,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1484
2129
|
dlq,
|
|
1485
2130
|
retry,
|
|
1486
2131
|
retryTopics: options.retryTopics,
|
|
1487
|
-
timeoutMs,
|
|
2132
|
+
timeoutMs: options.handlerTimeoutMs,
|
|
1488
2133
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1489
|
-
deduplication,
|
|
2134
|
+
deduplication: this.resolveDeduplicationContext(gid, options.deduplication),
|
|
1490
2135
|
messageTtlMs: options.messageTtlMs,
|
|
1491
2136
|
onTtlExpired: options.onTtlExpired,
|
|
1492
2137
|
eosMainContext
|
|
@@ -1497,70 +2142,24 @@ var KafkaClient = class _KafkaClient {
|
|
|
1497
2142
|
});
|
|
1498
2143
|
this.runningConsumers.set(gid, "eachMessage");
|
|
1499
2144
|
if (options.retryTopics && retry) {
|
|
1500
|
-
|
|
1501
|
-
await this.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
1502
|
-
}
|
|
1503
|
-
const companions = await startRetryTopicConsumers(
|
|
1504
|
-
topicNames,
|
|
1505
|
-
gid,
|
|
1506
|
-
handleMessage,
|
|
1507
|
-
retry,
|
|
1508
|
-
dlq,
|
|
1509
|
-
interceptors,
|
|
1510
|
-
schemaMap,
|
|
1511
|
-
this.retryTopicDeps,
|
|
1512
|
-
options.retryTopicAssignmentTimeoutMs
|
|
1513
|
-
);
|
|
1514
|
-
this.companionGroupIds.set(gid, companions);
|
|
2145
|
+
await this.launchRetryChain(gid, topicNames, handleMessage, retry, dlq, interceptors, schemaMap, options.retryTopicAssignmentTimeoutMs);
|
|
1515
2146
|
}
|
|
1516
2147
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
1517
2148
|
}
|
|
1518
2149
|
async startBatchConsumer(topics, handleBatch, options = {}) {
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1522
|
-
);
|
|
1523
|
-
}
|
|
1524
|
-
const hasRegexTopics = topics.some((t) => t instanceof RegExp);
|
|
1525
|
-
if (options.retryTopics && hasRegexTopics) {
|
|
1526
|
-
throw new Error(
|
|
1527
|
-
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
1528
|
-
);
|
|
1529
|
-
}
|
|
1530
|
-
if (options.retryTopics) {
|
|
1531
|
-
} else if (options.autoCommit !== false) {
|
|
2150
|
+
this.validateTopicConsumerOpts(topics, options);
|
|
2151
|
+
if (!options.retryTopics && options.autoCommit !== false) {
|
|
1532
2152
|
this.logger.debug?.(
|
|
1533
2153
|
`startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
|
|
1534
2154
|
);
|
|
1535
2155
|
}
|
|
1536
2156
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1537
2157
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", setupOptions);
|
|
1538
|
-
if (options.circuitBreaker)
|
|
1539
|
-
this.circuitConfigs.set(gid, options.circuitBreaker);
|
|
2158
|
+
if (options.circuitBreaker) this.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
1540
2159
|
const deps = this.messageDepsFor(gid);
|
|
1541
|
-
const
|
|
1542
|
-
const deduplication = this.resolveDeduplicationContext(
|
|
1543
|
-
gid,
|
|
1544
|
-
options.deduplication
|
|
1545
|
-
);
|
|
1546
|
-
let eosMainContext;
|
|
1547
|
-
if (options.retryTopics && retry) {
|
|
1548
|
-
const mainTxId = `${gid}-main-tx`;
|
|
1549
|
-
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1550
|
-
eosMainContext = { txProducer, consumer };
|
|
1551
|
-
}
|
|
2160
|
+
const eosMainContext = await this.makeEosMainContext(gid, consumer, options);
|
|
1552
2161
|
await consumer.run({
|
|
1553
|
-
|
|
1554
|
-
* eachBatch: called by the consumer for each batch of messages.
|
|
1555
|
-
* Called with the `payload` argument, which is an object containing the
|
|
1556
|
-
* batch of messages and a `BatchMeta` object with offset management controls.
|
|
1557
|
-
*
|
|
1558
|
-
* The function is wrapped with `trackInFlight` and `handleEachBatch` to provide
|
|
1559
|
-
* error handling and offset management.
|
|
1560
|
-
*
|
|
1561
|
-
* @param payload - an object containing the batch of messages and a `BatchMeta` object.
|
|
1562
|
-
*/
|
|
1563
|
-
eachBatch: (payload) => this.trackInFlight(
|
|
2162
|
+
eachBatch: (payload) => this.inFlight.track(
|
|
1564
2163
|
() => handleEachBatch(
|
|
1565
2164
|
payload,
|
|
1566
2165
|
{
|
|
@@ -1570,9 +2169,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1570
2169
|
dlq,
|
|
1571
2170
|
retry,
|
|
1572
2171
|
retryTopics: options.retryTopics,
|
|
1573
|
-
timeoutMs,
|
|
2172
|
+
timeoutMs: options.handlerTimeoutMs,
|
|
1574
2173
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1575
|
-
deduplication,
|
|
2174
|
+
deduplication: this.resolveDeduplicationContext(gid, options.deduplication),
|
|
1576
2175
|
messageTtlMs: options.messageTtlMs,
|
|
1577
2176
|
onTtlExpired: options.onTtlExpired,
|
|
1578
2177
|
eosMainContext
|
|
@@ -1583,9 +2182,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1583
2182
|
});
|
|
1584
2183
|
this.runningConsumers.set(gid, "eachBatch");
|
|
1585
2184
|
if (options.retryTopics && retry) {
|
|
1586
|
-
if (!this.autoCreateTopicsEnabled) {
|
|
1587
|
-
await this.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
1588
|
-
}
|
|
1589
2185
|
const handleMessageForRetry = (env) => handleBatch([env], {
|
|
1590
2186
|
partition: env.partition,
|
|
1591
2187
|
highWatermark: null,
|
|
@@ -1596,18 +2192,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1596
2192
|
commitOffsetsIfNecessary: async () => {
|
|
1597
2193
|
}
|
|
1598
2194
|
});
|
|
1599
|
-
|
|
1600
|
-
topicNames,
|
|
1601
|
-
gid,
|
|
1602
|
-
handleMessageForRetry,
|
|
1603
|
-
retry,
|
|
1604
|
-
dlq,
|
|
1605
|
-
interceptors,
|
|
1606
|
-
schemaMap,
|
|
1607
|
-
this.retryTopicDeps,
|
|
1608
|
-
options.retryTopicAssignmentTimeoutMs
|
|
1609
|
-
);
|
|
1610
|
-
this.companionGroupIds.set(gid, companions);
|
|
2195
|
+
await this.launchRetryChain(gid, topicNames, handleMessageForRetry, retry, dlq, interceptors, schemaMap, options.retryTopicAssignmentTimeoutMs);
|
|
1611
2196
|
}
|
|
1612
2197
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
1613
2198
|
}
|
|
@@ -1666,38 +2251,20 @@ var KafkaClient = class _KafkaClient {
|
|
|
1666
2251
|
if (groupId !== void 0) {
|
|
1667
2252
|
const consumer = this.consumers.get(groupId);
|
|
1668
2253
|
if (!consumer) {
|
|
1669
|
-
this.logger.warn(
|
|
1670
|
-
`stopConsumer: no active consumer for group "${groupId}"`
|
|
1671
|
-
);
|
|
2254
|
+
this.logger.warn(`stopConsumer: no active consumer for group "${groupId}"`);
|
|
1672
2255
|
return;
|
|
1673
2256
|
}
|
|
1674
|
-
await consumer.disconnect().catch(
|
|
1675
|
-
(e) => this.logger.warn(
|
|
1676
|
-
`Error disconnecting consumer "${groupId}":`,
|
|
1677
|
-
toError(e).message
|
|
1678
|
-
)
|
|
1679
|
-
);
|
|
2257
|
+
await consumer.disconnect().catch((e) => this.logger.warn(`Error disconnecting consumer "${groupId}":`, toError(e).message));
|
|
1680
2258
|
this.consumers.delete(groupId);
|
|
1681
2259
|
this.runningConsumers.delete(groupId);
|
|
1682
2260
|
this.consumerCreationOptions.delete(groupId);
|
|
1683
2261
|
this.dedupStates.delete(groupId);
|
|
1684
|
-
|
|
1685
|
-
if (key.startsWith(`${groupId}:`)) {
|
|
1686
|
-
clearTimeout(this.circuitStates.get(key).timer);
|
|
1687
|
-
this.circuitStates.delete(key);
|
|
1688
|
-
}
|
|
1689
|
-
}
|
|
1690
|
-
this.circuitConfigs.delete(groupId);
|
|
2262
|
+
this.circuitBreaker.removeGroup(groupId);
|
|
1691
2263
|
this.logger.log(`Consumer disconnected: group "${groupId}"`);
|
|
1692
2264
|
const mainTxId = `${groupId}-main-tx`;
|
|
1693
2265
|
const mainTxProducer = this.retryTxProducers.get(mainTxId);
|
|
1694
2266
|
if (mainTxProducer) {
|
|
1695
|
-
await mainTxProducer.disconnect().catch(
|
|
1696
|
-
(e) => this.logger.warn(
|
|
1697
|
-
`Error disconnecting main tx producer "${mainTxId}":`,
|
|
1698
|
-
toError(e).message
|
|
1699
|
-
)
|
|
1700
|
-
);
|
|
2267
|
+
await mainTxProducer.disconnect().catch((e) => this.logger.warn(`Error disconnecting main tx producer "${mainTxId}":`, toError(e).message));
|
|
1701
2268
|
_activeTransactionalIds.delete(mainTxId);
|
|
1702
2269
|
this.retryTxProducers.delete(mainTxId);
|
|
1703
2270
|
}
|
|
@@ -1705,12 +2272,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1705
2272
|
for (const cGroupId of companions) {
|
|
1706
2273
|
const cConsumer = this.consumers.get(cGroupId);
|
|
1707
2274
|
if (cConsumer) {
|
|
1708
|
-
await cConsumer.disconnect().catch(
|
|
1709
|
-
(e) => this.logger.warn(
|
|
1710
|
-
`Error disconnecting retry consumer "${cGroupId}":`,
|
|
1711
|
-
toError(e).message
|
|
1712
|
-
)
|
|
1713
|
-
);
|
|
2275
|
+
await cConsumer.disconnect().catch((e) => this.logger.warn(`Error disconnecting retry consumer "${cGroupId}":`, toError(e).message));
|
|
1714
2276
|
this.consumers.delete(cGroupId);
|
|
1715
2277
|
this.runningConsumers.delete(cGroupId);
|
|
1716
2278
|
this.consumerCreationOptions.delete(cGroupId);
|
|
@@ -1719,12 +2281,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1719
2281
|
const txId = `${cGroupId}-tx`;
|
|
1720
2282
|
const txProducer = this.retryTxProducers.get(txId);
|
|
1721
2283
|
if (txProducer) {
|
|
1722
|
-
await txProducer.disconnect().catch(
|
|
1723
|
-
(e) => this.logger.warn(
|
|
1724
|
-
`Error disconnecting retry tx producer "${txId}":`,
|
|
1725
|
-
toError(e).message
|
|
1726
|
-
)
|
|
1727
|
-
);
|
|
2284
|
+
await txProducer.disconnect().catch((e) => this.logger.warn(`Error disconnecting retry tx producer "${txId}":`, toError(e).message));
|
|
1728
2285
|
_activeTransactionalIds.delete(txId);
|
|
1729
2286
|
this.retryTxProducers.delete(txId);
|
|
1730
2287
|
}
|
|
@@ -1732,14 +2289,10 @@ var KafkaClient = class _KafkaClient {
|
|
|
1732
2289
|
this.companionGroupIds.delete(groupId);
|
|
1733
2290
|
} else {
|
|
1734
2291
|
const tasks = [
|
|
1735
|
-
...Array.from(this.consumers.values()).map(
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
)
|
|
1739
|
-
...Array.from(this.retryTxProducers.values()).map(
|
|
1740
|
-
(p) => p.disconnect().catch(() => {
|
|
1741
|
-
})
|
|
1742
|
-
)
|
|
2292
|
+
...Array.from(this.consumers.values()).map((c) => c.disconnect().catch(() => {
|
|
2293
|
+
})),
|
|
2294
|
+
...Array.from(this.retryTxProducers.values()).map((p) => p.disconnect().catch(() => {
|
|
2295
|
+
}))
|
|
1743
2296
|
];
|
|
1744
2297
|
await Promise.allSettled(tasks);
|
|
1745
2298
|
this.consumers.clear();
|
|
@@ -1748,10 +2301,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1748
2301
|
this.companionGroupIds.clear();
|
|
1749
2302
|
this.retryTxProducers.clear();
|
|
1750
2303
|
this.dedupStates.clear();
|
|
1751
|
-
|
|
1752
|
-
clearTimeout(state.timer);
|
|
1753
|
-
this.circuitStates.clear();
|
|
1754
|
-
this.circuitConfigs.clear();
|
|
2304
|
+
this.circuitBreaker.clear();
|
|
1755
2305
|
this.logger.log("All consumers disconnected");
|
|
1756
2306
|
}
|
|
1757
2307
|
}
|
|
@@ -1769,9 +2319,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1769
2319
|
return;
|
|
1770
2320
|
}
|
|
1771
2321
|
consumer.pause(
|
|
1772
|
-
assignments.flatMap(
|
|
1773
|
-
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1774
|
-
)
|
|
2322
|
+
assignments.flatMap(({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] })))
|
|
1775
2323
|
);
|
|
1776
2324
|
}
|
|
1777
2325
|
/**
|
|
@@ -1788,9 +2336,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1788
2336
|
return;
|
|
1789
2337
|
}
|
|
1790
2338
|
consumer.resume(
|
|
1791
|
-
assignments.flatMap(
|
|
1792
|
-
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1793
|
-
)
|
|
2339
|
+
assignments.flatMap(({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] })))
|
|
1794
2340
|
);
|
|
1795
2341
|
}
|
|
1796
2342
|
/** Pause all assigned partitions of a topic for a consumer group (used for queue backpressure). */
|
|
@@ -1811,14 +2357,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1811
2357
|
if (partitions.length > 0)
|
|
1812
2358
|
consumer.resume(partitions.map((p) => ({ topic: topic2, partitions: [p] })));
|
|
1813
2359
|
}
|
|
1814
|
-
/** DLQ header keys added by `sendToDlq` — stripped before re-publishing. */
|
|
1815
|
-
static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
1816
|
-
"x-dlq-original-topic",
|
|
1817
|
-
"x-dlq-failed-at",
|
|
1818
|
-
"x-dlq-error-message",
|
|
1819
|
-
"x-dlq-error-stack",
|
|
1820
|
-
"x-dlq-attempt-count"
|
|
1821
|
-
]);
|
|
1822
2360
|
/**
|
|
1823
2361
|
* Re-publish messages from a dead letter queue back to the original topic.
|
|
1824
2362
|
*
|
|
@@ -1831,106 +2369,27 @@ var KafkaClient = class _KafkaClient {
|
|
|
1831
2369
|
* @returns { replayed: number; skipped: number } - counts of re-published vs skipped messages
|
|
1832
2370
|
*/
|
|
1833
2371
|
async replayDlq(topic2, options = {}) {
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
}
|
|
1844
|
-
const highWatermarks = new Map(
|
|
1845
|
-
activePartitions.map(({ partition, high }) => [
|
|
1846
|
-
partition,
|
|
1847
|
-
parseInt(high, 10)
|
|
1848
|
-
])
|
|
1849
|
-
);
|
|
1850
|
-
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1851
|
-
let replayed = 0;
|
|
1852
|
-
let skipped = 0;
|
|
1853
|
-
const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
|
|
1854
|
-
await new Promise((resolve, reject) => {
|
|
1855
|
-
const consumer = getOrCreateConsumer(
|
|
1856
|
-
tempGroupId,
|
|
1857
|
-
true,
|
|
1858
|
-
true,
|
|
1859
|
-
this.consumerOpsDeps
|
|
1860
|
-
);
|
|
1861
|
-
const cleanup = () => {
|
|
2372
|
+
await this.adminOps.ensureConnected();
|
|
2373
|
+
return replayDlqTopic(topic2, options, {
|
|
2374
|
+
logger: this.logger,
|
|
2375
|
+
fetchTopicOffsets: (t) => this.adminOps.admin.fetchTopicOffsets(t),
|
|
2376
|
+
send: async (t, messages) => {
|
|
2377
|
+
await this.producer.send({ topic: t, messages });
|
|
2378
|
+
},
|
|
2379
|
+
createConsumer: (gid) => getOrCreateConsumer(gid, true, true, this._consumerOpsDeps),
|
|
2380
|
+
cleanupConsumer: (consumer, gid) => {
|
|
1862
2381
|
consumer.disconnect().catch(() => {
|
|
1863
2382
|
}).finally(() => {
|
|
1864
|
-
this.consumers.delete(
|
|
1865
|
-
this.runningConsumers.delete(
|
|
1866
|
-
this.consumerCreationOptions.delete(
|
|
2383
|
+
this.consumers.delete(gid);
|
|
2384
|
+
this.runningConsumers.delete(gid);
|
|
2385
|
+
this.consumerCreationOptions.delete(gid);
|
|
1867
2386
|
});
|
|
1868
|
-
}
|
|
1869
|
-
|
|
1870
|
-
() => consumer.run({
|
|
1871
|
-
eachMessage: async ({ partition, message }) => {
|
|
1872
|
-
if (!message.value) return;
|
|
1873
|
-
const offset = parseInt(message.offset, 10);
|
|
1874
|
-
processedOffsets.set(partition, offset);
|
|
1875
|
-
const headers = decodeHeaders(message.headers);
|
|
1876
|
-
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1877
|
-
const originalHeaders = Object.fromEntries(
|
|
1878
|
-
Object.entries(headers).filter(
|
|
1879
|
-
([k]) => !_KafkaClient.DLQ_HEADER_KEYS.has(k)
|
|
1880
|
-
)
|
|
1881
|
-
);
|
|
1882
|
-
const value = message.value.toString();
|
|
1883
|
-
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1884
|
-
if (!targetTopic || !shouldProcess) {
|
|
1885
|
-
skipped++;
|
|
1886
|
-
} else if (options.dryRun) {
|
|
1887
|
-
this.logger.log(
|
|
1888
|
-
`[DLQ replay dry-run] Would replay to "${targetTopic}"`
|
|
1889
|
-
);
|
|
1890
|
-
replayed++;
|
|
1891
|
-
} else {
|
|
1892
|
-
await this.producer.send({
|
|
1893
|
-
topic: targetTopic,
|
|
1894
|
-
messages: [{ value, headers: originalHeaders }]
|
|
1895
|
-
});
|
|
1896
|
-
replayed++;
|
|
1897
|
-
}
|
|
1898
|
-
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1899
|
-
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1900
|
-
);
|
|
1901
|
-
if (allDone) {
|
|
1902
|
-
cleanup();
|
|
1903
|
-
resolve();
|
|
1904
|
-
}
|
|
1905
|
-
}
|
|
1906
|
-
})
|
|
1907
|
-
).catch((err) => {
|
|
1908
|
-
cleanup();
|
|
1909
|
-
reject(err);
|
|
1910
|
-
});
|
|
2387
|
+
},
|
|
2388
|
+
dlqHeaderKeys: _KafkaClient.DLQ_HEADER_KEYS
|
|
1911
2389
|
});
|
|
1912
|
-
this.logger.log(
|
|
1913
|
-
`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`
|
|
1914
|
-
);
|
|
1915
|
-
return { replayed, skipped };
|
|
1916
2390
|
}
|
|
1917
2391
|
async resetOffsets(groupId, topic2, position) {
|
|
1918
|
-
|
|
1919
|
-
if (this.runningConsumers.has(gid)) {
|
|
1920
|
-
throw new Error(
|
|
1921
|
-
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
1922
|
-
);
|
|
1923
|
-
}
|
|
1924
|
-
await this.ensureAdminConnected();
|
|
1925
|
-
const partitionOffsets = await this.admin.fetchTopicOffsets(topic2);
|
|
1926
|
-
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
1927
|
-
partition,
|
|
1928
|
-
offset: position === "earliest" ? low : high
|
|
1929
|
-
}));
|
|
1930
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1931
|
-
this.logger.log(
|
|
1932
|
-
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1933
|
-
);
|
|
2392
|
+
return this.adminOps.resetOffsets(groupId, topic2, position);
|
|
1934
2393
|
}
|
|
1935
2394
|
/**
|
|
1936
2395
|
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
@@ -1938,25 +2397,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1938
2397
|
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1939
2398
|
*/
|
|
1940
2399
|
async seekToOffset(groupId, assignments) {
|
|
1941
|
-
|
|
1942
|
-
if (this.runningConsumers.has(gid)) {
|
|
1943
|
-
throw new Error(
|
|
1944
|
-
`seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1945
|
-
);
|
|
1946
|
-
}
|
|
1947
|
-
await this.ensureAdminConnected();
|
|
1948
|
-
const byTopic = /* @__PURE__ */ new Map();
|
|
1949
|
-
for (const { topic: topic2, partition, offset } of assignments) {
|
|
1950
|
-
const list = byTopic.get(topic2) ?? [];
|
|
1951
|
-
list.push({ partition, offset });
|
|
1952
|
-
byTopic.set(topic2, list);
|
|
1953
|
-
}
|
|
1954
|
-
for (const [topic2, partitions] of byTopic) {
|
|
1955
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1956
|
-
this.logger.log(
|
|
1957
|
-
`Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
|
|
1958
|
-
);
|
|
1959
|
-
}
|
|
2400
|
+
return this.adminOps.seekToOffset(groupId, assignments);
|
|
1960
2401
|
}
|
|
1961
2402
|
/**
|
|
1962
2403
|
* Seek specific topic-partition pairs to the offset nearest to a given timestamp
|
|
@@ -1967,37 +2408,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1967
2408
|
* future timestamp), the partition falls back to `-1` (end of topic — new messages only).
|
|
1968
2409
|
*/
|
|
1969
2410
|
async seekToTimestamp(groupId, assignments) {
|
|
1970
|
-
|
|
1971
|
-
if (this.runningConsumers.has(gid)) {
|
|
1972
|
-
throw new Error(
|
|
1973
|
-
`seekToTimestamp: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1974
|
-
);
|
|
1975
|
-
}
|
|
1976
|
-
await this.ensureAdminConnected();
|
|
1977
|
-
const byTopic = /* @__PURE__ */ new Map();
|
|
1978
|
-
for (const { topic: topic2, partition, timestamp } of assignments) {
|
|
1979
|
-
const list = byTopic.get(topic2) ?? [];
|
|
1980
|
-
list.push({ partition, timestamp });
|
|
1981
|
-
byTopic.set(topic2, list);
|
|
1982
|
-
}
|
|
1983
|
-
for (const [topic2, parts] of byTopic) {
|
|
1984
|
-
const offsets = await Promise.all(
|
|
1985
|
-
parts.map(async ({ partition, timestamp }) => {
|
|
1986
|
-
const results = await this.admin.fetchTopicOffsetsByTime(
|
|
1987
|
-
topic2,
|
|
1988
|
-
timestamp
|
|
1989
|
-
);
|
|
1990
|
-
const found = results.find(
|
|
1991
|
-
(r) => r.partition === partition
|
|
1992
|
-
);
|
|
1993
|
-
return { partition, offset: found?.offset ?? "-1" };
|
|
1994
|
-
})
|
|
1995
|
-
);
|
|
1996
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
|
|
1997
|
-
this.logger.log(
|
|
1998
|
-
`Offsets set by timestamp for group "${gid}" on "${topic2}": ${JSON.stringify(offsets)}`
|
|
1999
|
-
);
|
|
2000
|
-
}
|
|
2411
|
+
return this.adminOps.seekToTimestamp(groupId, assignments);
|
|
2001
2412
|
}
|
|
2002
2413
|
/**
|
|
2003
2414
|
* Returns the current circuit breaker state for a specific topic partition.
|
|
@@ -2011,14 +2422,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
2011
2422
|
* @returns `{ status, failures, windowSize }` snapshot for a given partition or `undefined` if no state exists.
|
|
2012
2423
|
*/
|
|
2013
2424
|
getCircuitState(topic2, partition, groupId) {
|
|
2014
|
-
|
|
2015
|
-
const state = this.circuitStates.get(`${gid}:${topic2}:${partition}`);
|
|
2016
|
-
if (!state) return void 0;
|
|
2017
|
-
return {
|
|
2018
|
-
status: state.status,
|
|
2019
|
-
failures: state.window.filter((v) => !v).length,
|
|
2020
|
-
windowSize: state.window.length
|
|
2021
|
-
};
|
|
2425
|
+
return this.circuitBreaker.getState(topic2, partition, groupId ?? this.defaultGroupId);
|
|
2022
2426
|
}
|
|
2023
2427
|
/**
|
|
2024
2428
|
* Query consumer group lag per partition.
|
|
@@ -2032,83 +2436,32 @@ var KafkaClient = class _KafkaClient {
|
|
|
2032
2436
|
* committed offset. Use `checkStatus()` to verify broker connectivity in that case.
|
|
2033
2437
|
*/
|
|
2034
2438
|
async getConsumerLag(groupId) {
|
|
2035
|
-
|
|
2036
|
-
await this.ensureAdminConnected();
|
|
2037
|
-
const committedByTopic = await this.admin.fetchOffsets({ groupId: gid });
|
|
2038
|
-
const brokerOffsetsAll = await Promise.all(
|
|
2039
|
-
committedByTopic.map(({ topic: topic2 }) => this.admin.fetchTopicOffsets(topic2))
|
|
2040
|
-
);
|
|
2041
|
-
const result = [];
|
|
2042
|
-
for (let i = 0; i < committedByTopic.length; i++) {
|
|
2043
|
-
const { topic: topic2, partitions } = committedByTopic[i];
|
|
2044
|
-
const brokerOffsets = brokerOffsetsAll[i];
|
|
2045
|
-
for (const { partition, offset } of partitions) {
|
|
2046
|
-
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
2047
|
-
if (!broker) continue;
|
|
2048
|
-
const committed = parseInt(offset, 10);
|
|
2049
|
-
const high = parseInt(broker.high, 10);
|
|
2050
|
-
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
2051
|
-
result.push({ topic: topic2, partition, lag });
|
|
2052
|
-
}
|
|
2053
|
-
}
|
|
2054
|
-
return result;
|
|
2439
|
+
return this.adminOps.getConsumerLag(groupId);
|
|
2055
2440
|
}
|
|
2056
2441
|
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
2057
2442
|
async checkStatus() {
|
|
2058
|
-
|
|
2059
|
-
await this.ensureAdminConnected();
|
|
2060
|
-
const topics = await this.admin.listTopics();
|
|
2061
|
-
return { status: "up", clientId: this.clientId, topics };
|
|
2062
|
-
} catch (error) {
|
|
2063
|
-
return {
|
|
2064
|
-
status: "down",
|
|
2065
|
-
clientId: this.clientId,
|
|
2066
|
-
error: error instanceof Error ? error.message : String(error)
|
|
2067
|
-
};
|
|
2068
|
-
}
|
|
2443
|
+
return this.adminOps.checkStatus();
|
|
2069
2444
|
}
|
|
2070
2445
|
/**
|
|
2071
2446
|
* List all consumer groups known to the broker.
|
|
2072
2447
|
* Useful for monitoring which groups are active and their current state.
|
|
2073
2448
|
*/
|
|
2074
2449
|
async listConsumerGroups() {
|
|
2075
|
-
|
|
2076
|
-
const result = await this.admin.listGroups();
|
|
2077
|
-
return result.groups.map((g) => ({
|
|
2078
|
-
groupId: g.groupId,
|
|
2079
|
-
state: g.state ?? "Unknown"
|
|
2080
|
-
}));
|
|
2450
|
+
return this.adminOps.listConsumerGroups();
|
|
2081
2451
|
}
|
|
2082
2452
|
/**
|
|
2083
2453
|
* Describe topics — returns partition layout, leader, replicas, and ISR.
|
|
2084
2454
|
* @param topics Topic names to describe. Omit to describe all topics.
|
|
2085
2455
|
*/
|
|
2086
2456
|
async describeTopics(topics) {
|
|
2087
|
-
|
|
2088
|
-
const result = await this.admin.fetchTopicMetadata(
|
|
2089
|
-
topics ? { topics } : void 0
|
|
2090
|
-
);
|
|
2091
|
-
return result.topics.map((t) => ({
|
|
2092
|
-
name: t.name,
|
|
2093
|
-
partitions: t.partitions.map((p) => ({
|
|
2094
|
-
partition: p.partitionId ?? p.partition,
|
|
2095
|
-
leader: p.leader,
|
|
2096
|
-
replicas: p.replicas.map(
|
|
2097
|
-
(r) => typeof r === "number" ? r : r.nodeId
|
|
2098
|
-
),
|
|
2099
|
-
isr: p.isr.map(
|
|
2100
|
-
(r) => typeof r === "number" ? r : r.nodeId
|
|
2101
|
-
)
|
|
2102
|
-
}))
|
|
2103
|
-
}));
|
|
2457
|
+
return this.adminOps.describeTopics(topics);
|
|
2104
2458
|
}
|
|
2105
2459
|
/**
|
|
2106
2460
|
* Delete records from a topic up to (but not including) the given offsets.
|
|
2107
2461
|
* All messages with offsets **before** the given offset are deleted.
|
|
2108
2462
|
*/
|
|
2109
2463
|
async deleteRecords(topic2, partitions) {
|
|
2110
|
-
|
|
2111
|
-
await this.admin.deleteTopicRecords({ topic: topic2, partitions });
|
|
2464
|
+
return this.adminOps.deleteRecords(topic2, partitions);
|
|
2112
2465
|
}
|
|
2113
2466
|
/** Return the client ID provided during `KafkaClient` construction. */
|
|
2114
2467
|
getClientId() {
|
|
@@ -2124,39 +2477,19 @@ var KafkaClient = class _KafkaClient {
|
|
|
2124
2477
|
* @returns Read-only `KafkaMetrics` snapshot: `processedCount`, `retryCount`, `dlqCount`, `dedupCount`.
|
|
2125
2478
|
*/
|
|
2126
2479
|
getMetrics(topic2) {
|
|
2127
|
-
|
|
2128
|
-
const m = this._topicMetrics.get(topic2);
|
|
2129
|
-
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
2130
|
-
}
|
|
2131
|
-
const agg = {
|
|
2132
|
-
processedCount: 0,
|
|
2133
|
-
retryCount: 0,
|
|
2134
|
-
dlqCount: 0,
|
|
2135
|
-
dedupCount: 0
|
|
2136
|
-
};
|
|
2137
|
-
for (const m of this._topicMetrics.values()) {
|
|
2138
|
-
agg.processedCount += m.processedCount;
|
|
2139
|
-
agg.retryCount += m.retryCount;
|
|
2140
|
-
agg.dlqCount += m.dlqCount;
|
|
2141
|
-
agg.dedupCount += m.dedupCount;
|
|
2142
|
-
}
|
|
2143
|
-
return agg;
|
|
2480
|
+
return this.metrics.getMetrics(topic2);
|
|
2144
2481
|
}
|
|
2145
2482
|
/**
|
|
2146
2483
|
* Reset internal event counters to zero.
|
|
2147
2484
|
*
|
|
2148
2485
|
* @param topic Topic name to reset. When omitted, all topics are reset.
|
|
2149
|
-
*/
|
|
2150
|
-
resetMetrics(topic2) {
|
|
2151
|
-
|
|
2152
|
-
this._topicMetrics.delete(topic2);
|
|
2153
|
-
return;
|
|
2154
|
-
}
|
|
2155
|
-
this._topicMetrics.clear();
|
|
2486
|
+
*/
|
|
2487
|
+
resetMetrics(topic2) {
|
|
2488
|
+
this.metrics.resetMetrics(topic2);
|
|
2156
2489
|
}
|
|
2157
2490
|
/** Gracefully disconnect producer, all consumers, and admin. */
|
|
2158
2491
|
async disconnect(drainTimeoutMs = 3e4) {
|
|
2159
|
-
await this.waitForDrain(drainTimeoutMs);
|
|
2492
|
+
await this.inFlight.waitForDrain(drainTimeoutMs);
|
|
2160
2493
|
const tasks = [this.producer.disconnect()];
|
|
2161
2494
|
if (this.txProducer) {
|
|
2162
2495
|
tasks.push(this.txProducer.disconnect());
|
|
@@ -2164,28 +2497,17 @@ var KafkaClient = class _KafkaClient {
|
|
|
2164
2497
|
this.txProducer = void 0;
|
|
2165
2498
|
this.txProducerInitPromise = void 0;
|
|
2166
2499
|
}
|
|
2167
|
-
for (const txId of this.retryTxProducers.keys())
|
|
2168
|
-
|
|
2169
|
-
}
|
|
2170
|
-
for (const p of this.retryTxProducers.values()) {
|
|
2171
|
-
tasks.push(p.disconnect());
|
|
2172
|
-
}
|
|
2500
|
+
for (const txId of this.retryTxProducers.keys()) _activeTransactionalIds.delete(txId);
|
|
2501
|
+
for (const p of this.retryTxProducers.values()) tasks.push(p.disconnect());
|
|
2173
2502
|
this.retryTxProducers.clear();
|
|
2174
|
-
for (const consumer of this.consumers.values())
|
|
2175
|
-
|
|
2176
|
-
}
|
|
2177
|
-
if (this.isAdminConnected) {
|
|
2178
|
-
tasks.push(this.admin.disconnect());
|
|
2179
|
-
this.isAdminConnected = false;
|
|
2180
|
-
}
|
|
2503
|
+
for (const consumer of this.consumers.values()) tasks.push(consumer.disconnect());
|
|
2504
|
+
tasks.push(this.adminOps.disconnect());
|
|
2181
2505
|
await Promise.allSettled(tasks);
|
|
2182
2506
|
this.consumers.clear();
|
|
2183
2507
|
this.runningConsumers.clear();
|
|
2184
2508
|
this.consumerCreationOptions.clear();
|
|
2185
2509
|
this.companionGroupIds.clear();
|
|
2186
|
-
|
|
2187
|
-
this.circuitStates.clear();
|
|
2188
|
-
this.circuitConfigs.clear();
|
|
2510
|
+
this.circuitBreaker.clear();
|
|
2189
2511
|
this.logger.log("All connections closed");
|
|
2190
2512
|
}
|
|
2191
2513
|
// ── Graceful shutdown ────────────────────────────────────────────
|
|
@@ -2204,235 +2526,20 @@ var KafkaClient = class _KafkaClient {
|
|
|
2204
2526
|
*/
|
|
2205
2527
|
enableGracefulShutdown(signals = ["SIGTERM", "SIGINT"], drainTimeoutMs = 3e4) {
|
|
2206
2528
|
const handler = () => {
|
|
2207
|
-
this.logger.log(
|
|
2208
|
-
"Shutdown signal received \u2014 draining in-flight handlers..."
|
|
2209
|
-
);
|
|
2529
|
+
this.logger.log("Shutdown signal received \u2014 draining in-flight handlers...");
|
|
2210
2530
|
this.disconnect(drainTimeoutMs).catch(
|
|
2211
|
-
(err) => this.logger.error(
|
|
2212
|
-
"Error during graceful shutdown:",
|
|
2213
|
-
toError(err).message
|
|
2214
|
-
)
|
|
2531
|
+
(err) => this.logger.error("Error during graceful shutdown:", toError(err).message)
|
|
2215
2532
|
);
|
|
2216
2533
|
};
|
|
2217
|
-
for (const signal of signals)
|
|
2218
|
-
process.once(signal, handler);
|
|
2219
|
-
}
|
|
2220
|
-
}
|
|
2221
|
-
/**
|
|
2222
|
-
* Increment the in-flight handler count and return a promise that calls the given handler.
|
|
2223
|
-
* When the promise resolves or rejects, decrement the in flight handler count.
|
|
2224
|
-
* If the in flight handler count reaches 0, call all previously registered drain resolvers.
|
|
2225
|
-
* @param fn The handler to call when the promise is resolved or rejected.
|
|
2226
|
-
* @returns A promise that resolves or rejects with the result of calling the handler.
|
|
2227
|
-
*/
|
|
2228
|
-
trackInFlight(fn) {
|
|
2229
|
-
this.inFlightTotal++;
|
|
2230
|
-
return fn().finally(() => {
|
|
2231
|
-
this.inFlightTotal--;
|
|
2232
|
-
if (this.inFlightTotal === 0) {
|
|
2233
|
-
this.drainResolvers.splice(0).forEach((r) => r());
|
|
2234
|
-
}
|
|
2235
|
-
});
|
|
2236
|
-
}
|
|
2237
|
-
/**
|
|
2238
|
-
* Waits for all in-flight handlers to complete or for a given timeout, whichever comes first.
|
|
2239
|
-
* @param timeoutMs Maximum time to wait in milliseconds.
|
|
2240
|
-
* @returns A promise that resolves when all handlers have completed or the timeout is reached.
|
|
2241
|
-
* @private
|
|
2242
|
-
*/
|
|
2243
|
-
waitForDrain(timeoutMs) {
|
|
2244
|
-
if (this.inFlightTotal === 0) return Promise.resolve();
|
|
2245
|
-
return new Promise((resolve) => {
|
|
2246
|
-
let handle;
|
|
2247
|
-
const onDrain = () => {
|
|
2248
|
-
clearTimeout(handle);
|
|
2249
|
-
resolve();
|
|
2250
|
-
};
|
|
2251
|
-
this.drainResolvers.push(onDrain);
|
|
2252
|
-
handle = setTimeout(() => {
|
|
2253
|
-
const idx = this.drainResolvers.indexOf(onDrain);
|
|
2254
|
-
if (idx !== -1) this.drainResolvers.splice(idx, 1);
|
|
2255
|
-
this.logger.warn(
|
|
2256
|
-
`Drain timed out after ${timeoutMs}ms \u2014 ${this.inFlightTotal} handler(s) still in flight`
|
|
2257
|
-
);
|
|
2258
|
-
resolve();
|
|
2259
|
-
}, timeoutMs);
|
|
2260
|
-
});
|
|
2534
|
+
for (const signal of signals) process.once(signal, handler);
|
|
2261
2535
|
}
|
|
2262
2536
|
// ── Private helpers ──────────────────────────────────────────────
|
|
2263
|
-
/**
|
|
2264
|
-
* Prepare a send payload by registering the topic's schema and then building the payload.
|
|
2265
|
-
* @param topicOrDesc - topic name or topic descriptor
|
|
2266
|
-
* @param messages - batch of messages to send
|
|
2267
|
-
* @returns - prepared payload
|
|
2268
|
-
*/
|
|
2269
2537
|
async preparePayload(topicOrDesc, messages, compression) {
|
|
2270
2538
|
registerSchema(topicOrDesc, this.schemaRegistry, this.logger);
|
|
2271
|
-
const payload = await buildSendPayload(
|
|
2272
|
-
topicOrDesc,
|
|
2273
|
-
messages,
|
|
2274
|
-
this.producerOpsDeps,
|
|
2275
|
-
compression
|
|
2276
|
-
);
|
|
2539
|
+
const payload = await buildSendPayload(topicOrDesc, messages, this._producerOpsDeps, compression);
|
|
2277
2540
|
await this.ensureTopic(payload.topic);
|
|
2278
2541
|
return payload;
|
|
2279
2542
|
}
|
|
2280
|
-
// afterSend is called once per message — symmetric with beforeSend in buildSendPayload.
|
|
2281
|
-
notifyAfterSend(topic2, count) {
|
|
2282
|
-
for (let i = 0; i < count; i++) {
|
|
2283
|
-
for (const inst of this.instrumentation) {
|
|
2284
|
-
inst.afterSend?.(topic2);
|
|
2285
|
-
}
|
|
2286
|
-
}
|
|
2287
|
-
}
|
|
2288
|
-
/**
|
|
2289
|
-
* Returns the KafkaMetrics for a given topic.
|
|
2290
|
-
* If the topic hasn't seen any events, initializes a zero-valued snapshot.
|
|
2291
|
-
* @param topic - name of the topic to get the metrics for
|
|
2292
|
-
* @returns - KafkaMetrics for the given topic
|
|
2293
|
-
*/
|
|
2294
|
-
metricsFor(topic2) {
|
|
2295
|
-
let m = this._topicMetrics.get(topic2);
|
|
2296
|
-
if (!m) {
|
|
2297
|
-
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
2298
|
-
this._topicMetrics.set(topic2, m);
|
|
2299
|
-
}
|
|
2300
|
-
return m;
|
|
2301
|
-
}
|
|
2302
|
-
/**
|
|
2303
|
-
* Notifies instrumentation hooks of a retry event.
|
|
2304
|
-
* @param envelope The original message envelope that triggered the retry.
|
|
2305
|
-
* @param attempt The current retry attempt (1-indexed).
|
|
2306
|
-
* @param maxRetries The maximum number of retries configured for this topic.
|
|
2307
|
-
*/
|
|
2308
|
-
notifyRetry(envelope, attempt, maxRetries) {
|
|
2309
|
-
this.metricsFor(envelope.topic).retryCount++;
|
|
2310
|
-
for (const inst of this.instrumentation) {
|
|
2311
|
-
inst.onRetry?.(envelope, attempt, maxRetries);
|
|
2312
|
-
}
|
|
2313
|
-
}
|
|
2314
|
-
/**
|
|
2315
|
-
* Called whenever a message is routed to the dead letter queue.
|
|
2316
|
-
* @param envelope The original message envelope.
|
|
2317
|
-
* @param reason The reason for routing to the dead letter queue.
|
|
2318
|
-
* @param gid The group ID of the consumer that triggered the circuit breaker, if any.
|
|
2319
|
-
*/
|
|
2320
|
-
notifyDlq(envelope, reason, gid) {
|
|
2321
|
-
this.metricsFor(envelope.topic).dlqCount++;
|
|
2322
|
-
for (const inst of this.instrumentation) {
|
|
2323
|
-
inst.onDlq?.(envelope, reason);
|
|
2324
|
-
}
|
|
2325
|
-
if (!gid) return;
|
|
2326
|
-
const cfg = this.circuitConfigs.get(gid);
|
|
2327
|
-
if (!cfg) return;
|
|
2328
|
-
const threshold = cfg.threshold ?? 5;
|
|
2329
|
-
const recoveryMs = cfg.recoveryMs ?? 3e4;
|
|
2330
|
-
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
2331
|
-
let state = this.circuitStates.get(stateKey);
|
|
2332
|
-
if (!state) {
|
|
2333
|
-
state = { status: "closed", window: [], successes: 0 };
|
|
2334
|
-
this.circuitStates.set(stateKey, state);
|
|
2335
|
-
}
|
|
2336
|
-
if (state.status === "open") return;
|
|
2337
|
-
const openCircuit = () => {
|
|
2338
|
-
state.status = "open";
|
|
2339
|
-
state.window = [];
|
|
2340
|
-
state.successes = 0;
|
|
2341
|
-
clearTimeout(state.timer);
|
|
2342
|
-
for (const inst of this.instrumentation)
|
|
2343
|
-
inst.onCircuitOpen?.(envelope.topic, envelope.partition);
|
|
2344
|
-
this.pauseConsumer(gid, [
|
|
2345
|
-
{ topic: envelope.topic, partitions: [envelope.partition] }
|
|
2346
|
-
]);
|
|
2347
|
-
state.timer = setTimeout(() => {
|
|
2348
|
-
state.status = "half-open";
|
|
2349
|
-
state.successes = 0;
|
|
2350
|
-
this.logger.log(
|
|
2351
|
-
`[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2352
|
-
);
|
|
2353
|
-
for (const inst of this.instrumentation)
|
|
2354
|
-
inst.onCircuitHalfOpen?.(envelope.topic, envelope.partition);
|
|
2355
|
-
this.resumeConsumer(gid, [
|
|
2356
|
-
{ topic: envelope.topic, partitions: [envelope.partition] }
|
|
2357
|
-
]);
|
|
2358
|
-
}, recoveryMs);
|
|
2359
|
-
};
|
|
2360
|
-
if (state.status === "half-open") {
|
|
2361
|
-
clearTimeout(state.timer);
|
|
2362
|
-
this.logger.warn(
|
|
2363
|
-
`[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2364
|
-
);
|
|
2365
|
-
openCircuit();
|
|
2366
|
-
return;
|
|
2367
|
-
}
|
|
2368
|
-
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
2369
|
-
state.window = [...state.window, false];
|
|
2370
|
-
if (state.window.length > windowSize) {
|
|
2371
|
-
state.window = state.window.slice(state.window.length - windowSize);
|
|
2372
|
-
}
|
|
2373
|
-
const failures = state.window.filter((v) => !v).length;
|
|
2374
|
-
if (failures >= threshold) {
|
|
2375
|
-
this.logger.warn(
|
|
2376
|
-
`[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
|
|
2377
|
-
);
|
|
2378
|
-
openCircuit();
|
|
2379
|
-
}
|
|
2380
|
-
}
|
|
2381
|
-
/**
|
|
2382
|
-
* Notify all instrumentation hooks about a duplicate message detection.
|
|
2383
|
-
* Invoked by the consumer after a message has been successfully processed
|
|
2384
|
-
* and the Lamport clock detected a duplicate.
|
|
2385
|
-
* @param envelope The processed message envelope.
|
|
2386
|
-
* @param strategy The duplicate detection strategy used.
|
|
2387
|
-
*/
|
|
2388
|
-
notifyDuplicate(envelope, strategy) {
|
|
2389
|
-
this.metricsFor(envelope.topic).dedupCount++;
|
|
2390
|
-
for (const inst of this.instrumentation) {
|
|
2391
|
-
inst.onDuplicate?.(envelope, strategy);
|
|
2392
|
-
}
|
|
2393
|
-
}
|
|
2394
|
-
/**
|
|
2395
|
-
* Notify all instrumentation hooks about a successfully processed message.
|
|
2396
|
-
* Invoked by the consumer after a message has been successfully processed
|
|
2397
|
-
* by the handler.
|
|
2398
|
-
* @param envelope The processed message envelope.
|
|
2399
|
-
* @param gid The optional consumer group ID.
|
|
2400
|
-
*/
|
|
2401
|
-
notifyMessage(envelope, gid) {
|
|
2402
|
-
this.metricsFor(envelope.topic).processedCount++;
|
|
2403
|
-
for (const inst of this.instrumentation) {
|
|
2404
|
-
inst.onMessage?.(envelope);
|
|
2405
|
-
}
|
|
2406
|
-
if (!gid) return;
|
|
2407
|
-
const cfg = this.circuitConfigs.get(gid);
|
|
2408
|
-
if (!cfg) return;
|
|
2409
|
-
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
2410
|
-
const state = this.circuitStates.get(stateKey);
|
|
2411
|
-
if (!state) return;
|
|
2412
|
-
const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
|
|
2413
|
-
if (state.status === "half-open") {
|
|
2414
|
-
state.successes++;
|
|
2415
|
-
if (state.successes >= halfOpenSuccesses) {
|
|
2416
|
-
clearTimeout(state.timer);
|
|
2417
|
-
state.timer = void 0;
|
|
2418
|
-
state.status = "closed";
|
|
2419
|
-
state.window = [];
|
|
2420
|
-
state.successes = 0;
|
|
2421
|
-
this.logger.log(
|
|
2422
|
-
`[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2423
|
-
);
|
|
2424
|
-
for (const inst of this.instrumentation)
|
|
2425
|
-
inst.onCircuitClose?.(envelope.topic, envelope.partition);
|
|
2426
|
-
}
|
|
2427
|
-
} else if (state.status === "closed") {
|
|
2428
|
-
const threshold = cfg.threshold ?? 5;
|
|
2429
|
-
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
2430
|
-
state.window = [...state.window, true];
|
|
2431
|
-
if (state.window.length > windowSize) {
|
|
2432
|
-
state.window = state.window.slice(state.window.length - windowSize);
|
|
2433
|
-
}
|
|
2434
|
-
}
|
|
2435
|
-
}
|
|
2436
2543
|
/**
|
|
2437
2544
|
* Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
|
|
2438
2545
|
* The handler itself is not cancelled — the warning is diagnostic only.
|
|
@@ -2443,79 +2550,10 @@ var KafkaClient = class _KafkaClient {
|
|
|
2443
2550
|
if (timer !== void 0) clearTimeout(timer);
|
|
2444
2551
|
});
|
|
2445
2552
|
timer = setTimeout(() => {
|
|
2446
|
-
this.logger.warn(
|
|
2447
|
-
`Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
|
|
2448
|
-
);
|
|
2553
|
+
this.logger.warn(`Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`);
|
|
2449
2554
|
}, timeoutMs);
|
|
2450
2555
|
return promise;
|
|
2451
2556
|
}
|
|
2452
|
-
/**
|
|
2453
|
-
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
2454
|
-
* `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
|
|
2455
|
-
* rather than silently discovering missing topics on the first handler failure.
|
|
2456
|
-
*/
|
|
2457
|
-
async validateRetryTopicsExist(topicNames, maxRetries) {
|
|
2458
|
-
await this.ensureAdminConnected();
|
|
2459
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2460
|
-
const missing = [];
|
|
2461
|
-
for (const t of topicNames) {
|
|
2462
|
-
for (let level = 1; level <= maxRetries; level++) {
|
|
2463
|
-
const retryTopic = `${t}.retry.${level}`;
|
|
2464
|
-
if (!existing.has(retryTopic)) missing.push(retryTopic);
|
|
2465
|
-
}
|
|
2466
|
-
}
|
|
2467
|
-
if (missing.length > 0) {
|
|
2468
|
-
throw new Error(
|
|
2469
|
-
`retryTopics: true but the following retry topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2470
|
-
);
|
|
2471
|
-
}
|
|
2472
|
-
}
|
|
2473
|
-
/**
|
|
2474
|
-
* When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
|
|
2475
|
-
* consumed topic. Throws a clear error at startup rather than silently discovering
|
|
2476
|
-
* missing DLQ topics on the first handler failure.
|
|
2477
|
-
*/
|
|
2478
|
-
async validateDlqTopicsExist(topicNames) {
|
|
2479
|
-
await this.ensureAdminConnected();
|
|
2480
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2481
|
-
const missing = topicNames.filter((t) => !existing.has(`${t}.dlq`)).map((t) => `${t}.dlq`);
|
|
2482
|
-
if (missing.length > 0) {
|
|
2483
|
-
throw new Error(
|
|
2484
|
-
`dlq: true but the following DLQ topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2485
|
-
);
|
|
2486
|
-
}
|
|
2487
|
-
}
|
|
2488
|
-
/**
|
|
2489
|
-
* When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
|
|
2490
|
-
* that every `<topic>.duplicates` destination topic already exists. Throws a
|
|
2491
|
-
* clear error at startup rather than silently dropping duplicates on first hit.
|
|
2492
|
-
*/
|
|
2493
|
-
async validateDuplicatesTopicsExist(topicNames, customDestination) {
|
|
2494
|
-
await this.ensureAdminConnected();
|
|
2495
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2496
|
-
const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
|
|
2497
|
-
const missing = toCheck.filter((t) => !existing.has(t));
|
|
2498
|
-
if (missing.length > 0) {
|
|
2499
|
-
throw new Error(
|
|
2500
|
-
`deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2501
|
-
);
|
|
2502
|
-
}
|
|
2503
|
-
}
|
|
2504
|
-
/**
|
|
2505
|
-
* Connect the admin client if not already connected.
|
|
2506
|
-
* The flag is only set to `true` after a successful connect — if `admin.connect()`
|
|
2507
|
-
* throws the flag remains `false` so the next call will retry the connection.
|
|
2508
|
-
*/
|
|
2509
|
-
async ensureAdminConnected() {
|
|
2510
|
-
if (this.isAdminConnected) return;
|
|
2511
|
-
try {
|
|
2512
|
-
await this.admin.connect();
|
|
2513
|
-
this.isAdminConnected = true;
|
|
2514
|
-
} catch (err) {
|
|
2515
|
-
this.isAdminConnected = false;
|
|
2516
|
-
throw err;
|
|
2517
|
-
}
|
|
2518
|
-
}
|
|
2519
2557
|
/**
|
|
2520
2558
|
* Create and connect a transactional producer for EOS retry routing.
|
|
2521
2559
|
* Each retry level consumer gets its own producer with a unique `transactionalId`
|
|
@@ -2528,12 +2566,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
2528
2566
|
);
|
|
2529
2567
|
}
|
|
2530
2568
|
const p = this.kafka.producer({
|
|
2531
|
-
kafkaJS: {
|
|
2532
|
-
acks: -1,
|
|
2533
|
-
idempotent: true,
|
|
2534
|
-
transactionalId,
|
|
2535
|
-
maxInFlightRequests: 1
|
|
2536
|
-
}
|
|
2569
|
+
kafkaJS: { acks: -1, idempotent: true, transactionalId, maxInFlightRequests: 1 }
|
|
2537
2570
|
});
|
|
2538
2571
|
await p.connect();
|
|
2539
2572
|
_activeTransactionalIds.add(transactionalId);
|
|
@@ -2542,20 +2575,16 @@ var KafkaClient = class _KafkaClient {
|
|
|
2542
2575
|
}
|
|
2543
2576
|
/**
|
|
2544
2577
|
* Ensure that a topic exists by creating it if it doesn't already exist.
|
|
2545
|
-
* If `autoCreateTopics` is disabled,
|
|
2546
|
-
*
|
|
2547
|
-
* If multiple concurrent calls are made to `ensureTopic` for the same topic,
|
|
2548
|
-
* they are deduplicated to prevent multiple calls to `admin.createTopics()`.
|
|
2549
|
-
* @param topic - The topic to ensure exists.
|
|
2550
|
-
* @returns A promise that resolves when the topic has been created or already exists.
|
|
2578
|
+
* If `autoCreateTopics` is disabled, returns immediately.
|
|
2579
|
+
* Concurrent calls for the same topic are deduplicated.
|
|
2551
2580
|
*/
|
|
2552
2581
|
async ensureTopic(topic2) {
|
|
2553
2582
|
if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
|
|
2554
2583
|
let p = this.ensureTopicPromises.get(topic2);
|
|
2555
2584
|
if (!p) {
|
|
2556
2585
|
p = (async () => {
|
|
2557
|
-
await this.
|
|
2558
|
-
await this.admin.createTopics({
|
|
2586
|
+
await this.adminOps.ensureConnected();
|
|
2587
|
+
await this.adminOps.admin.createTopics({
|
|
2559
2588
|
topics: [{ topic: topic2, numPartitions: this.numPartitions }]
|
|
2560
2589
|
});
|
|
2561
2590
|
this.ensuredTopics.add(topic2);
|
|
@@ -2595,99 +2624,78 @@ var KafkaClient = class _KafkaClient {
|
|
|
2595
2624
|
gid,
|
|
2596
2625
|
fromBeginning,
|
|
2597
2626
|
options.autoCommit ?? true,
|
|
2598
|
-
this.
|
|
2627
|
+
this._consumerOpsDeps,
|
|
2599
2628
|
options.partitionAssigner
|
|
2600
2629
|
);
|
|
2601
|
-
const schemaMap = buildSchemaMap(
|
|
2602
|
-
stringTopics,
|
|
2603
|
-
this.schemaRegistry,
|
|
2604
|
-
optionSchemas,
|
|
2605
|
-
this.logger
|
|
2606
|
-
);
|
|
2630
|
+
const schemaMap = buildSchemaMap(stringTopics, this.schemaRegistry, optionSchemas, this.logger);
|
|
2607
2631
|
const topicNames = stringTopics.map((t) => resolveTopicName(t));
|
|
2608
|
-
const subscribeTopics = [
|
|
2609
|
-
|
|
2610
|
-
...regexTopics
|
|
2611
|
-
];
|
|
2612
|
-
for (const t of topicNames) {
|
|
2613
|
-
await this.ensureTopic(t);
|
|
2614
|
-
}
|
|
2632
|
+
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2633
|
+
for (const t of topicNames) await this.ensureTopic(t);
|
|
2615
2634
|
if (dlq) {
|
|
2616
|
-
for (const t of topicNames) {
|
|
2617
|
-
await this.ensureTopic(`${t}.dlq`);
|
|
2618
|
-
}
|
|
2635
|
+
for (const t of topicNames) await this.ensureTopic(`${t}.dlq`);
|
|
2619
2636
|
if (!this.autoCreateTopicsEnabled && topicNames.length > 0) {
|
|
2620
|
-
await this.validateDlqTopicsExist(topicNames);
|
|
2637
|
+
await this.adminOps.validateDlqTopicsExist(topicNames);
|
|
2621
2638
|
}
|
|
2622
2639
|
}
|
|
2623
2640
|
if (options.deduplication?.strategy === "topic") {
|
|
2624
2641
|
const dest = options.deduplication.duplicatesTopic;
|
|
2625
2642
|
if (this.autoCreateTopicsEnabled) {
|
|
2626
|
-
for (const t of topicNames) {
|
|
2627
|
-
await this.ensureTopic(dest ?? `${t}.duplicates`);
|
|
2628
|
-
}
|
|
2643
|
+
for (const t of topicNames) await this.ensureTopic(dest ?? `${t}.duplicates`);
|
|
2629
2644
|
} else if (topicNames.length > 0) {
|
|
2630
|
-
await this.validateDuplicatesTopicsExist(topicNames, dest);
|
|
2645
|
+
await this.adminOps.validateDuplicatesTopicsExist(topicNames, dest);
|
|
2631
2646
|
}
|
|
2632
2647
|
}
|
|
2633
2648
|
await consumer.connect();
|
|
2634
|
-
await subscribeWithRetry(
|
|
2635
|
-
consumer,
|
|
2636
|
-
subscribeTopics,
|
|
2637
|
-
this.logger,
|
|
2638
|
-
options.subscribeRetry
|
|
2639
|
-
);
|
|
2649
|
+
await subscribeWithRetry(consumer, subscribeTopics, this.logger, options.subscribeRetry);
|
|
2640
2650
|
const displayTopics = subscribeTopics.map((t) => t instanceof RegExp ? t.toString() : t).join(", ");
|
|
2641
|
-
this.logger.log(
|
|
2642
|
-
`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
|
|
2643
|
-
);
|
|
2651
|
+
this.logger.log(`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`);
|
|
2644
2652
|
return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex };
|
|
2645
2653
|
}
|
|
2646
2654
|
/** Create or retrieve the deduplication context for a consumer group. */
|
|
2647
2655
|
resolveDeduplicationContext(groupId, options) {
|
|
2648
2656
|
if (!options) return void 0;
|
|
2649
|
-
if (!this.dedupStates.has(groupId))
|
|
2650
|
-
this.dedupStates.set(groupId, /* @__PURE__ */ new Map());
|
|
2651
|
-
}
|
|
2657
|
+
if (!this.dedupStates.has(groupId)) this.dedupStates.set(groupId, /* @__PURE__ */ new Map());
|
|
2652
2658
|
return { options, state: this.dedupStates.get(groupId) };
|
|
2653
2659
|
}
|
|
2654
|
-
// ──
|
|
2655
|
-
/**
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
strictSchemasEnabled: this.strictSchemasEnabled,
|
|
2668
|
-
instrumentation: this.instrumentation,
|
|
2669
|
-
logger: this.logger,
|
|
2670
|
-
nextLamportClock: () => ++this._lamportClock
|
|
2671
|
-
};
|
|
2660
|
+
// ── Shared consumer setup helpers ────────────────────────────────
|
|
2661
|
+
/** Guard checks shared by startConsumer and startBatchConsumer. */
|
|
2662
|
+
validateTopicConsumerOpts(topics, options) {
|
|
2663
|
+
if (options.retryTopics && !options.retry) {
|
|
2664
|
+
throw new Error(
|
|
2665
|
+
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
2666
|
+
);
|
|
2667
|
+
}
|
|
2668
|
+
if (options.retryTopics && topics.some((t) => t instanceof RegExp)) {
|
|
2669
|
+
throw new Error(
|
|
2670
|
+
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
2671
|
+
);
|
|
2672
|
+
}
|
|
2672
2673
|
}
|
|
2673
|
-
/**
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2674
|
+
/** Create EOS transactional producer context for atomic main → retry.1 routing. */
|
|
2675
|
+
async makeEosMainContext(gid, consumer, options) {
|
|
2676
|
+
if (!options.retryTopics || !options.retry) return void 0;
|
|
2677
|
+
const txProducer = await this.createRetryTxProducer(`${gid}-main-tx`);
|
|
2678
|
+
return { txProducer, consumer };
|
|
2679
|
+
}
|
|
2680
|
+
/** Start companion retry-level consumers and register them under the main groupId. */
|
|
2681
|
+
async launchRetryChain(gid, topicNames, handleMessage, retry, dlq, interceptors, schemaMap, assignmentTimeoutMs) {
|
|
2682
|
+
if (!this.autoCreateTopicsEnabled) {
|
|
2683
|
+
await this.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
2684
|
+
}
|
|
2685
|
+
const companions = await startRetryTopicConsumers(
|
|
2686
|
+
topicNames,
|
|
2687
|
+
gid,
|
|
2688
|
+
handleMessage,
|
|
2689
|
+
retry,
|
|
2690
|
+
dlq,
|
|
2691
|
+
interceptors,
|
|
2692
|
+
schemaMap,
|
|
2693
|
+
this._retryTopicDeps,
|
|
2694
|
+
assignmentTimeoutMs
|
|
2695
|
+
);
|
|
2696
|
+
this.companionGroupIds.set(gid, companions);
|
|
2690
2697
|
}
|
|
2698
|
+
// ── Deps object builders ─────────────────────────────────────────
|
|
2691
2699
|
/** Build MessageHandlerDeps with circuit breaker callbacks bound to the given groupId. */
|
|
2692
2700
|
messageDepsFor(gid) {
|
|
2693
2701
|
return {
|
|
@@ -2696,38 +2704,24 @@ var KafkaClient = class _KafkaClient {
|
|
|
2696
2704
|
instrumentation: this.instrumentation,
|
|
2697
2705
|
onMessageLost: this.onMessageLost,
|
|
2698
2706
|
onTtlExpired: this.onTtlExpired,
|
|
2699
|
-
onRetry: this.notifyRetry.bind(this),
|
|
2700
|
-
onDlq: (envelope, reason) => this.notifyDlq(envelope, reason, gid),
|
|
2701
|
-
onDuplicate: this.notifyDuplicate.bind(this),
|
|
2702
|
-
onMessage: (envelope) => this.notifyMessage(envelope, gid)
|
|
2707
|
+
onRetry: this.metrics.notifyRetry.bind(this.metrics),
|
|
2708
|
+
onDlq: (envelope, reason) => this.metrics.notifyDlq(envelope, reason, gid),
|
|
2709
|
+
onDuplicate: this.metrics.notifyDuplicate.bind(this.metrics),
|
|
2710
|
+
onMessage: (envelope) => this.metrics.notifyMessage(envelope, gid)
|
|
2703
2711
|
};
|
|
2704
2712
|
}
|
|
2705
|
-
/**
|
|
2706
|
-
|
|
2707
|
-
*
|
|
2708
|
-
* `logger`: The logger instance passed to the retry topic consumers.
|
|
2709
|
-
* `producer`: The producer instance passed to the retry topic consumers.
|
|
2710
|
-
* `instrumentation`: The instrumentation instance passed to the retry topic consumers.
|
|
2711
|
-
* `onMessageLost`: The callback function passed to the retry topic consumers for tracking lost messages.
|
|
2712
|
-
* `onRetry`: The callback function passed to the retry topic consumers for tracking retry attempts.
|
|
2713
|
-
* `onDlq`: The callback function passed to the retry topic consumers for tracking dead-letter queue routing.
|
|
2714
|
-
* `onMessage`: The callback function passed to the retry topic consumers for tracking message delivery.
|
|
2715
|
-
* `ensureTopic`: A function that ensures a topic exists before subscribing to it.
|
|
2716
|
-
* `getOrCreateConsumer`: A function that creates or retrieves a consumer instance.
|
|
2717
|
-
* `runningConsumers`: A map of consumer group IDs to their corresponding consumer instances.
|
|
2718
|
-
* `createRetryTxProducer`: A function that creates a retry transactional producer instance.
|
|
2719
|
-
*/
|
|
2720
|
-
get retryTopicDeps() {
|
|
2713
|
+
/** Build the deps object passed to retry topic consumers. */
|
|
2714
|
+
buildRetryTopicDeps() {
|
|
2721
2715
|
return {
|
|
2722
2716
|
logger: this.logger,
|
|
2723
2717
|
producer: this.producer,
|
|
2724
2718
|
instrumentation: this.instrumentation,
|
|
2725
2719
|
onMessageLost: this.onMessageLost,
|
|
2726
|
-
onRetry: this.notifyRetry.bind(this),
|
|
2727
|
-
onDlq: this.notifyDlq.bind(this),
|
|
2728
|
-
onMessage: this.notifyMessage.bind(this),
|
|
2720
|
+
onRetry: this.metrics.notifyRetry.bind(this.metrics),
|
|
2721
|
+
onDlq: this.metrics.notifyDlq.bind(this.metrics),
|
|
2722
|
+
onMessage: this.metrics.notifyMessage.bind(this.metrics),
|
|
2729
2723
|
ensureTopic: (t) => this.ensureTopic(t),
|
|
2730
|
-
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.
|
|
2724
|
+
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this._consumerOpsDeps),
|
|
2731
2725
|
runningConsumers: this.runningConsumers,
|
|
2732
2726
|
createRetryTxProducer: (txId) => this.createRetryTxProducer(txId)
|
|
2733
2727
|
};
|
|
@@ -2768,4 +2762,4 @@ export {
|
|
|
2768
2762
|
KafkaClient,
|
|
2769
2763
|
topic
|
|
2770
2764
|
};
|
|
2771
|
-
//# sourceMappingURL=chunk-
|
|
2765
|
+
//# sourceMappingURL=chunk-BVWRZTMD.mjs.map
|