@drarzter/kafka-client 0.7.2 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/{chunk-MADAJD2F.mjs → chunk-XP7LLRGQ.mjs} +751 -861
- package/dist/chunk-XP7LLRGQ.mjs.map +1 -0
- package/dist/core.d.mts +19 -137
- package/dist/core.d.ts +19 -137
- package/dist/core.js +750 -860
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +1 -1
- package/dist/index.js +751 -861
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
- package/dist/chunk-MADAJD2F.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -153,7 +153,7 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
|
153
153
|
}
|
|
154
154
|
};
|
|
155
155
|
|
|
156
|
-
// src/client/kafka.client/producer
|
|
156
|
+
// src/client/kafka.client/producer/ops.ts
|
|
157
157
|
function resolveTopicName(topicOrDescriptor) {
|
|
158
158
|
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
159
159
|
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
@@ -231,7 +231,7 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
231
231
|
return { topic: topic2, messages: builtMessages, ...compression && { compression } };
|
|
232
232
|
}
|
|
233
233
|
|
|
234
|
-
// src/client/kafka.client/consumer
|
|
234
|
+
// src/client/kafka.client/consumer/ops.ts
|
|
235
235
|
var import_kafka_javascript = require("@confluentinc/kafka-javascript");
|
|
236
236
|
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner) {
|
|
237
237
|
const { consumers, consumerCreationOptions, kafka, onRebalance, logger } = deps;
|
|
@@ -294,7 +294,7 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
|
|
|
294
294
|
return schemaMap;
|
|
295
295
|
}
|
|
296
296
|
|
|
297
|
-
// src/client/consumer/pipeline.ts
|
|
297
|
+
// src/client/kafka.client/consumer/pipeline.ts
|
|
298
298
|
function toError(error) {
|
|
299
299
|
return error instanceof Error ? error : new Error(String(error));
|
|
300
300
|
}
|
|
@@ -644,7 +644,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
644
644
|
}
|
|
645
645
|
}
|
|
646
646
|
|
|
647
|
-
// src/client/kafka.client/
|
|
647
|
+
// src/client/kafka.client/consumer/handler.ts
|
|
648
648
|
async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
649
649
|
const clockRaw = envelope.headers[HEADER_LAMPORT_CLOCK];
|
|
650
650
|
if (clockRaw === void 0) return false;
|
|
@@ -965,7 +965,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
965
965
|
);
|
|
966
966
|
}
|
|
967
967
|
|
|
968
|
-
// src/client/consumer/subscribe-retry.ts
|
|
968
|
+
// src/client/kafka.client/consumer/subscribe-retry.ts
|
|
969
969
|
async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
970
970
|
const maxAttempts = retryOpts?.retries ?? 5;
|
|
971
971
|
const backoffMs = retryOpts?.backoffMs ?? 5e3;
|
|
@@ -986,7 +986,7 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
|
986
986
|
}
|
|
987
987
|
}
|
|
988
988
|
|
|
989
|
-
// src/client/kafka.client/retry-topic.ts
|
|
989
|
+
// src/client/kafka.client/consumer/retry-topic.ts
|
|
990
990
|
async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs = 1e4) {
|
|
991
991
|
const topicSet = new Set(topics);
|
|
992
992
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -1240,9 +1240,7 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
1240
1240
|
return levelGroupIds;
|
|
1241
1241
|
}
|
|
1242
1242
|
|
|
1243
|
-
// src/client/kafka.client/
|
|
1244
|
-
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = import_kafka_javascript2.KafkaJS;
|
|
1245
|
-
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1243
|
+
// src/client/kafka.client/consumer/queue.ts
|
|
1246
1244
|
var AsyncQueue = class {
|
|
1247
1245
|
constructor(highWaterMark = Infinity, onFull = () => {
|
|
1248
1246
|
}, onDrained = () => {
|
|
@@ -1291,6 +1289,562 @@ var AsyncQueue = class {
|
|
|
1291
1289
|
return new Promise((resolve, reject) => this.waiting.push({ resolve, reject }));
|
|
1292
1290
|
}
|
|
1293
1291
|
};
|
|
1292
|
+
|
|
1293
|
+
// src/client/kafka.client/infra/circuit-breaker.ts
|
|
1294
|
+
var CircuitBreakerManager = class {
|
|
1295
|
+
constructor(deps) {
|
|
1296
|
+
this.deps = deps;
|
|
1297
|
+
}
|
|
1298
|
+
states = /* @__PURE__ */ new Map();
|
|
1299
|
+
configs = /* @__PURE__ */ new Map();
|
|
1300
|
+
setConfig(gid, options) {
|
|
1301
|
+
this.configs.set(gid, options);
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Returns a snapshot of the circuit breaker state for a given topic-partition.
|
|
1305
|
+
* Returns `undefined` when no state exists for the key.
|
|
1306
|
+
*/
|
|
1307
|
+
getState(topic2, partition, gid) {
|
|
1308
|
+
const state = this.states.get(`${gid}:${topic2}:${partition}`);
|
|
1309
|
+
if (!state) return void 0;
|
|
1310
|
+
return {
|
|
1311
|
+
status: state.status,
|
|
1312
|
+
failures: state.window.filter((v) => !v).length,
|
|
1313
|
+
windowSize: state.window.length
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
/**
|
|
1317
|
+
* Record a failure for the given envelope and group.
|
|
1318
|
+
* Drives the CLOSED → OPEN and HALF-OPEN → OPEN transitions.
|
|
1319
|
+
*/
|
|
1320
|
+
onFailure(envelope, gid) {
|
|
1321
|
+
const cfg = this.configs.get(gid);
|
|
1322
|
+
if (!cfg) return;
|
|
1323
|
+
const threshold = cfg.threshold ?? 5;
|
|
1324
|
+
const recoveryMs = cfg.recoveryMs ?? 3e4;
|
|
1325
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
1326
|
+
let state = this.states.get(stateKey);
|
|
1327
|
+
if (!state) {
|
|
1328
|
+
state = { status: "closed", window: [], successes: 0 };
|
|
1329
|
+
this.states.set(stateKey, state);
|
|
1330
|
+
}
|
|
1331
|
+
if (state.status === "open") return;
|
|
1332
|
+
const openCircuit = () => {
|
|
1333
|
+
state.status = "open";
|
|
1334
|
+
state.window = [];
|
|
1335
|
+
state.successes = 0;
|
|
1336
|
+
clearTimeout(state.timer);
|
|
1337
|
+
for (const inst of this.deps.instrumentation)
|
|
1338
|
+
inst.onCircuitOpen?.(envelope.topic, envelope.partition);
|
|
1339
|
+
this.deps.pauseConsumer(gid, [{ topic: envelope.topic, partitions: [envelope.partition] }]);
|
|
1340
|
+
state.timer = setTimeout(() => {
|
|
1341
|
+
state.status = "half-open";
|
|
1342
|
+
state.successes = 0;
|
|
1343
|
+
this.deps.logger.log(
|
|
1344
|
+
`[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1345
|
+
);
|
|
1346
|
+
for (const inst of this.deps.instrumentation)
|
|
1347
|
+
inst.onCircuitHalfOpen?.(envelope.topic, envelope.partition);
|
|
1348
|
+
this.deps.resumeConsumer(gid, [{ topic: envelope.topic, partitions: [envelope.partition] }]);
|
|
1349
|
+
}, recoveryMs);
|
|
1350
|
+
};
|
|
1351
|
+
if (state.status === "half-open") {
|
|
1352
|
+
clearTimeout(state.timer);
|
|
1353
|
+
this.deps.logger.warn(
|
|
1354
|
+
`[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1355
|
+
);
|
|
1356
|
+
openCircuit();
|
|
1357
|
+
return;
|
|
1358
|
+
}
|
|
1359
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
1360
|
+
state.window = [...state.window, false];
|
|
1361
|
+
if (state.window.length > windowSize) {
|
|
1362
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
1363
|
+
}
|
|
1364
|
+
const failures = state.window.filter((v) => !v).length;
|
|
1365
|
+
if (failures >= threshold) {
|
|
1366
|
+
this.deps.logger.warn(
|
|
1367
|
+
`[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
|
|
1368
|
+
);
|
|
1369
|
+
openCircuit();
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
/**
|
|
1373
|
+
* Record a success for the given envelope and group.
|
|
1374
|
+
* Drives the HALF-OPEN → CLOSED transition and updates the success window.
|
|
1375
|
+
*/
|
|
1376
|
+
onSuccess(envelope, gid) {
|
|
1377
|
+
const cfg = this.configs.get(gid);
|
|
1378
|
+
if (!cfg) return;
|
|
1379
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
1380
|
+
const state = this.states.get(stateKey);
|
|
1381
|
+
if (!state) return;
|
|
1382
|
+
const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
|
|
1383
|
+
if (state.status === "half-open") {
|
|
1384
|
+
state.successes++;
|
|
1385
|
+
if (state.successes >= halfOpenSuccesses) {
|
|
1386
|
+
clearTimeout(state.timer);
|
|
1387
|
+
state.timer = void 0;
|
|
1388
|
+
state.status = "closed";
|
|
1389
|
+
state.window = [];
|
|
1390
|
+
state.successes = 0;
|
|
1391
|
+
this.deps.logger.log(
|
|
1392
|
+
`[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1393
|
+
);
|
|
1394
|
+
for (const inst of this.deps.instrumentation)
|
|
1395
|
+
inst.onCircuitClose?.(envelope.topic, envelope.partition);
|
|
1396
|
+
}
|
|
1397
|
+
} else if (state.status === "closed") {
|
|
1398
|
+
const threshold = cfg.threshold ?? 5;
|
|
1399
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
1400
|
+
state.window = [...state.window, true];
|
|
1401
|
+
if (state.window.length > windowSize) {
|
|
1402
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
/**
|
|
1407
|
+
* Remove all circuit state and config for the given group.
|
|
1408
|
+
* Called when a consumer is stopped via `stopConsumer(groupId)`.
|
|
1409
|
+
*/
|
|
1410
|
+
removeGroup(gid) {
|
|
1411
|
+
for (const key of [...this.states.keys()]) {
|
|
1412
|
+
if (key.startsWith(`${gid}:`)) {
|
|
1413
|
+
clearTimeout(this.states.get(key).timer);
|
|
1414
|
+
this.states.delete(key);
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
this.configs.delete(gid);
|
|
1418
|
+
}
|
|
1419
|
+
/** Clear all circuit state and config. Called on `disconnect()`. */
|
|
1420
|
+
clear() {
|
|
1421
|
+
for (const state of this.states.values()) clearTimeout(state.timer);
|
|
1422
|
+
this.states.clear();
|
|
1423
|
+
this.configs.clear();
|
|
1424
|
+
}
|
|
1425
|
+
};
|
|
1426
|
+
|
|
1427
|
+
// src/client/kafka.client/admin/ops.ts
|
|
1428
|
+
var AdminOps = class {
|
|
1429
|
+
constructor(deps) {
|
|
1430
|
+
this.deps = deps;
|
|
1431
|
+
}
|
|
1432
|
+
isConnected = false;
|
|
1433
|
+
/** Underlying admin client — used by index.ts for topic validation. */
|
|
1434
|
+
get admin() {
|
|
1435
|
+
return this.deps.admin;
|
|
1436
|
+
}
|
|
1437
|
+
/** Whether the admin client is currently connected. */
|
|
1438
|
+
get connected() {
|
|
1439
|
+
return this.isConnected;
|
|
1440
|
+
}
|
|
1441
|
+
/**
|
|
1442
|
+
* Connect the admin client if not already connected.
|
|
1443
|
+
* The flag is only set to `true` after a successful connect — if `admin.connect()`
|
|
1444
|
+
* throws the flag remains `false` so the next call will retry the connection.
|
|
1445
|
+
*/
|
|
1446
|
+
async ensureConnected() {
|
|
1447
|
+
if (this.isConnected) return;
|
|
1448
|
+
try {
|
|
1449
|
+
await this.deps.admin.connect();
|
|
1450
|
+
this.isConnected = true;
|
|
1451
|
+
} catch (err) {
|
|
1452
|
+
this.isConnected = false;
|
|
1453
|
+
throw err;
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
/** Disconnect admin if connected. Resets the connected flag. */
|
|
1457
|
+
async disconnect() {
|
|
1458
|
+
if (!this.isConnected) return;
|
|
1459
|
+
await this.deps.admin.disconnect();
|
|
1460
|
+
this.isConnected = false;
|
|
1461
|
+
}
|
|
1462
|
+
async resetOffsets(groupId, topic2, position) {
|
|
1463
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1464
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1465
|
+
throw new Error(
|
|
1466
|
+
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
1467
|
+
);
|
|
1468
|
+
}
|
|
1469
|
+
await this.ensureConnected();
|
|
1470
|
+
const partitionOffsets = await this.deps.admin.fetchTopicOffsets(topic2);
|
|
1471
|
+
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
1472
|
+
partition,
|
|
1473
|
+
offset: position === "earliest" ? low : high
|
|
1474
|
+
}));
|
|
1475
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1476
|
+
this.deps.logger.log(
|
|
1477
|
+
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1478
|
+
);
|
|
1479
|
+
}
|
|
1480
|
+
/**
|
|
1481
|
+
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
1482
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
1483
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1484
|
+
*/
|
|
1485
|
+
async seekToOffset(groupId, assignments) {
|
|
1486
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1487
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1488
|
+
throw new Error(
|
|
1489
|
+
`seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1490
|
+
);
|
|
1491
|
+
}
|
|
1492
|
+
await this.ensureConnected();
|
|
1493
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
1494
|
+
for (const { topic: topic2, partition, offset } of assignments) {
|
|
1495
|
+
const list = byTopic.get(topic2) ?? [];
|
|
1496
|
+
list.push({ partition, offset });
|
|
1497
|
+
byTopic.set(topic2, list);
|
|
1498
|
+
}
|
|
1499
|
+
for (const [topic2, partitions] of byTopic) {
|
|
1500
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1501
|
+
this.deps.logger.log(
|
|
1502
|
+
`Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
|
|
1503
|
+
);
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
/**
|
|
1507
|
+
* Seek specific topic-partition pairs to the offset nearest to a given timestamp
|
|
1508
|
+
* (in milliseconds) for a stopped consumer group.
|
|
1509
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
1510
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1511
|
+
* If no offset exists at the requested timestamp (e.g. empty partition or
|
|
1512
|
+
* future timestamp), the partition falls back to `-1` (end of topic — new messages only).
|
|
1513
|
+
*/
|
|
1514
|
+
async seekToTimestamp(groupId, assignments) {
|
|
1515
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1516
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1517
|
+
throw new Error(
|
|
1518
|
+
`seekToTimestamp: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1519
|
+
);
|
|
1520
|
+
}
|
|
1521
|
+
await this.ensureConnected();
|
|
1522
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
1523
|
+
for (const { topic: topic2, partition, timestamp } of assignments) {
|
|
1524
|
+
const list = byTopic.get(topic2) ?? [];
|
|
1525
|
+
list.push({ partition, timestamp });
|
|
1526
|
+
byTopic.set(topic2, list);
|
|
1527
|
+
}
|
|
1528
|
+
for (const [topic2, parts] of byTopic) {
|
|
1529
|
+
const offsets = await Promise.all(
|
|
1530
|
+
parts.map(async ({ partition, timestamp }) => {
|
|
1531
|
+
const results = await this.deps.admin.fetchTopicOffsetsByTime(
|
|
1532
|
+
topic2,
|
|
1533
|
+
timestamp
|
|
1534
|
+
);
|
|
1535
|
+
const found = results.find(
|
|
1536
|
+
(r) => r.partition === partition
|
|
1537
|
+
);
|
|
1538
|
+
return { partition, offset: found?.offset ?? "-1" };
|
|
1539
|
+
})
|
|
1540
|
+
);
|
|
1541
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
|
|
1542
|
+
this.deps.logger.log(
|
|
1543
|
+
`Offsets set by timestamp for group "${gid}" on "${topic2}": ${JSON.stringify(offsets)}`
|
|
1544
|
+
);
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
/**
|
|
1548
|
+
* Query consumer group lag per partition.
|
|
1549
|
+
* Lag = broker high-watermark − last committed offset.
|
|
1550
|
+
* A committed offset of -1 (nothing committed yet) counts as full lag.
|
|
1551
|
+
*
|
|
1552
|
+
* Returns an empty array when the consumer group has never committed any
|
|
1553
|
+
* offsets (freshly created group, `autoCommit: false` with no manual commits,
|
|
1554
|
+
* or group not yet assigned). This is a Kafka protocol limitation:
|
|
1555
|
+
* `fetchOffsets` only returns data for topic-partitions that have at least one
|
|
1556
|
+
* committed offset. Use `checkStatus()` to verify broker connectivity in that case.
|
|
1557
|
+
*/
|
|
1558
|
+
async getConsumerLag(groupId) {
|
|
1559
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1560
|
+
await this.ensureConnected();
|
|
1561
|
+
const committedByTopic = await this.deps.admin.fetchOffsets({ groupId: gid });
|
|
1562
|
+
const brokerOffsetsAll = await Promise.all(
|
|
1563
|
+
committedByTopic.map(({ topic: topic2 }) => this.deps.admin.fetchTopicOffsets(topic2))
|
|
1564
|
+
);
|
|
1565
|
+
const result = [];
|
|
1566
|
+
for (let i = 0; i < committedByTopic.length; i++) {
|
|
1567
|
+
const { topic: topic2, partitions } = committedByTopic[i];
|
|
1568
|
+
const brokerOffsets = brokerOffsetsAll[i];
|
|
1569
|
+
for (const { partition, offset } of partitions) {
|
|
1570
|
+
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
1571
|
+
if (!broker) continue;
|
|
1572
|
+
const committed = parseInt(offset, 10);
|
|
1573
|
+
const high = parseInt(broker.high, 10);
|
|
1574
|
+
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
1575
|
+
result.push({ topic: topic2, partition, lag });
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
return result;
|
|
1579
|
+
}
|
|
1580
|
+
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
1581
|
+
async checkStatus() {
|
|
1582
|
+
try {
|
|
1583
|
+
await this.ensureConnected();
|
|
1584
|
+
const topics = await this.deps.admin.listTopics();
|
|
1585
|
+
return { status: "up", clientId: this.deps.clientId, topics };
|
|
1586
|
+
} catch (error) {
|
|
1587
|
+
return {
|
|
1588
|
+
status: "down",
|
|
1589
|
+
clientId: this.deps.clientId,
|
|
1590
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1591
|
+
};
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
/**
|
|
1595
|
+
* List all consumer groups known to the broker.
|
|
1596
|
+
* Useful for monitoring which groups are active and their current state.
|
|
1597
|
+
*/
|
|
1598
|
+
async listConsumerGroups() {
|
|
1599
|
+
await this.ensureConnected();
|
|
1600
|
+
const result = await this.deps.admin.listGroups();
|
|
1601
|
+
return result.groups.map((g) => ({
|
|
1602
|
+
groupId: g.groupId,
|
|
1603
|
+
state: g.state ?? "Unknown"
|
|
1604
|
+
}));
|
|
1605
|
+
}
|
|
1606
|
+
/**
|
|
1607
|
+
* Describe topics — returns partition layout, leader, replicas, and ISR.
|
|
1608
|
+
* @param topics Topic names to describe. Omit to describe all topics.
|
|
1609
|
+
*/
|
|
1610
|
+
async describeTopics(topics) {
|
|
1611
|
+
await this.ensureConnected();
|
|
1612
|
+
const result = await this.deps.admin.fetchTopicMetadata(
|
|
1613
|
+
topics ? { topics } : void 0
|
|
1614
|
+
);
|
|
1615
|
+
return result.topics.map((t) => ({
|
|
1616
|
+
name: t.name,
|
|
1617
|
+
partitions: t.partitions.map((p) => ({
|
|
1618
|
+
partition: p.partitionId ?? p.partition,
|
|
1619
|
+
leader: p.leader,
|
|
1620
|
+
replicas: p.replicas.map(
|
|
1621
|
+
(r) => typeof r === "number" ? r : r.nodeId
|
|
1622
|
+
),
|
|
1623
|
+
isr: p.isr.map(
|
|
1624
|
+
(r) => typeof r === "number" ? r : r.nodeId
|
|
1625
|
+
)
|
|
1626
|
+
}))
|
|
1627
|
+
}));
|
|
1628
|
+
}
|
|
1629
|
+
/**
|
|
1630
|
+
* Delete records from a topic up to (but not including) the given offsets.
|
|
1631
|
+
* All messages with offsets **before** the given offset are deleted.
|
|
1632
|
+
*/
|
|
1633
|
+
async deleteRecords(topic2, partitions) {
|
|
1634
|
+
await this.ensureConnected();
|
|
1635
|
+
await this.deps.admin.deleteTopicRecords({ topic: topic2, partitions });
|
|
1636
|
+
}
|
|
1637
|
+
/**
|
|
1638
|
+
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
1639
|
+
* `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
|
|
1640
|
+
* rather than silently discovering missing topics on the first handler failure.
|
|
1641
|
+
*/
|
|
1642
|
+
async validateRetryTopicsExist(topicNames, maxRetries) {
|
|
1643
|
+
await this.ensureConnected();
|
|
1644
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1645
|
+
const missing = [];
|
|
1646
|
+
for (const t of topicNames) {
|
|
1647
|
+
for (let level = 1; level <= maxRetries; level++) {
|
|
1648
|
+
const retryTopic = `${t}.retry.${level}`;
|
|
1649
|
+
if (!existing.has(retryTopic)) missing.push(retryTopic);
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
if (missing.length > 0) {
|
|
1653
|
+
throw new Error(
|
|
1654
|
+
`retryTopics: true but the following retry topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1655
|
+
);
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
/**
|
|
1659
|
+
* When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
|
|
1660
|
+
* consumed topic. Throws a clear error at startup rather than silently discovering
|
|
1661
|
+
* missing DLQ topics on the first handler failure.
|
|
1662
|
+
*/
|
|
1663
|
+
async validateDlqTopicsExist(topicNames) {
|
|
1664
|
+
await this.ensureConnected();
|
|
1665
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1666
|
+
const missing = topicNames.filter((t) => !existing.has(`${t}.dlq`)).map((t) => `${t}.dlq`);
|
|
1667
|
+
if (missing.length > 0) {
|
|
1668
|
+
throw new Error(
|
|
1669
|
+
`dlq: true but the following DLQ topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1670
|
+
);
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
/**
|
|
1674
|
+
* When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
|
|
1675
|
+
* that every `<topic>.duplicates` destination topic already exists. Throws a
|
|
1676
|
+
* clear error at startup rather than silently dropping duplicates on first hit.
|
|
1677
|
+
*/
|
|
1678
|
+
async validateDuplicatesTopicsExist(topicNames, customDestination) {
|
|
1679
|
+
await this.ensureConnected();
|
|
1680
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1681
|
+
const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
|
|
1682
|
+
const missing = toCheck.filter((t) => !existing.has(t));
|
|
1683
|
+
if (missing.length > 0) {
|
|
1684
|
+
throw new Error(
|
|
1685
|
+
`deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1686
|
+
);
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
};
|
|
1690
|
+
|
|
1691
|
+
// src/client/kafka.client/consumer/dlq-replay.ts
|
|
1692
|
+
async function replayDlqTopic(topic2, options = {}, deps) {
|
|
1693
|
+
const dlqTopic = `${topic2}.dlq`;
|
|
1694
|
+
const partitionOffsets = await deps.fetchTopicOffsets(dlqTopic);
|
|
1695
|
+
const activePartitions = partitionOffsets.filter((p) => parseInt(p.high, 10) > 0);
|
|
1696
|
+
if (activePartitions.length === 0) {
|
|
1697
|
+
deps.logger.log(`replayDlq: "${dlqTopic}" is empty \u2014 nothing to replay`);
|
|
1698
|
+
return { replayed: 0, skipped: 0 };
|
|
1699
|
+
}
|
|
1700
|
+
const highWatermarks = new Map(
|
|
1701
|
+
activePartitions.map(({ partition, high }) => [partition, parseInt(high, 10)])
|
|
1702
|
+
);
|
|
1703
|
+
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1704
|
+
let replayed = 0;
|
|
1705
|
+
let skipped = 0;
|
|
1706
|
+
const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
|
|
1707
|
+
await new Promise((resolve, reject) => {
|
|
1708
|
+
const consumer = deps.createConsumer(tempGroupId);
|
|
1709
|
+
const cleanup = () => deps.cleanupConsumer(consumer, tempGroupId);
|
|
1710
|
+
consumer.connect().then(() => subscribeWithRetry(consumer, [dlqTopic], deps.logger)).then(
|
|
1711
|
+
() => consumer.run({
|
|
1712
|
+
eachMessage: async ({ partition, message }) => {
|
|
1713
|
+
if (!message.value) return;
|
|
1714
|
+
const offset = parseInt(message.offset, 10);
|
|
1715
|
+
processedOffsets.set(partition, offset);
|
|
1716
|
+
const headers = decodeHeaders(message.headers);
|
|
1717
|
+
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1718
|
+
const originalHeaders = Object.fromEntries(
|
|
1719
|
+
Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
|
|
1720
|
+
);
|
|
1721
|
+
const value = message.value.toString();
|
|
1722
|
+
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1723
|
+
if (!targetTopic || !shouldProcess) {
|
|
1724
|
+
skipped++;
|
|
1725
|
+
} else if (options.dryRun) {
|
|
1726
|
+
deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
|
|
1727
|
+
replayed++;
|
|
1728
|
+
} else {
|
|
1729
|
+
await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
|
|
1730
|
+
replayed++;
|
|
1731
|
+
}
|
|
1732
|
+
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1733
|
+
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1734
|
+
);
|
|
1735
|
+
if (allDone) {
|
|
1736
|
+
cleanup();
|
|
1737
|
+
resolve();
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
})
|
|
1741
|
+
).catch((err) => {
|
|
1742
|
+
cleanup();
|
|
1743
|
+
reject(err);
|
|
1744
|
+
});
|
|
1745
|
+
});
|
|
1746
|
+
deps.logger.log(`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`);
|
|
1747
|
+
return { replayed, skipped };
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
// src/client/kafka.client/infra/metrics-manager.ts
|
|
1751
|
+
var MetricsManager = class {
|
|
1752
|
+
constructor(deps) {
|
|
1753
|
+
this.deps = deps;
|
|
1754
|
+
}
|
|
1755
|
+
topicMetrics = /* @__PURE__ */ new Map();
|
|
1756
|
+
metricsFor(topic2) {
|
|
1757
|
+
let m = this.topicMetrics.get(topic2);
|
|
1758
|
+
if (!m) {
|
|
1759
|
+
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1760
|
+
this.topicMetrics.set(topic2, m);
|
|
1761
|
+
}
|
|
1762
|
+
return m;
|
|
1763
|
+
}
|
|
1764
|
+
/** Fire `afterSend` instrumentation hooks for each message in a batch. */
|
|
1765
|
+
notifyAfterSend(topic2, count) {
|
|
1766
|
+
for (let i = 0; i < count; i++)
|
|
1767
|
+
for (const inst of this.deps.instrumentation) inst.afterSend?.(topic2);
|
|
1768
|
+
}
|
|
1769
|
+
notifyRetry(envelope, attempt, maxRetries) {
|
|
1770
|
+
this.metricsFor(envelope.topic).retryCount++;
|
|
1771
|
+
for (const inst of this.deps.instrumentation) inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1772
|
+
}
|
|
1773
|
+
notifyDlq(envelope, reason, gid) {
|
|
1774
|
+
this.metricsFor(envelope.topic).dlqCount++;
|
|
1775
|
+
for (const inst of this.deps.instrumentation) inst.onDlq?.(envelope, reason);
|
|
1776
|
+
if (gid) this.deps.onCircuitFailure(envelope, gid);
|
|
1777
|
+
}
|
|
1778
|
+
notifyDuplicate(envelope, strategy) {
|
|
1779
|
+
this.metricsFor(envelope.topic).dedupCount++;
|
|
1780
|
+
for (const inst of this.deps.instrumentation) inst.onDuplicate?.(envelope, strategy);
|
|
1781
|
+
}
|
|
1782
|
+
notifyMessage(envelope, gid) {
|
|
1783
|
+
this.metricsFor(envelope.topic).processedCount++;
|
|
1784
|
+
for (const inst of this.deps.instrumentation) inst.onMessage?.(envelope);
|
|
1785
|
+
if (gid) this.deps.onCircuitSuccess(envelope, gid);
|
|
1786
|
+
}
|
|
1787
|
+
getMetrics(topic2) {
|
|
1788
|
+
if (topic2 !== void 0) {
|
|
1789
|
+
const m = this.topicMetrics.get(topic2);
|
|
1790
|
+
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1791
|
+
}
|
|
1792
|
+
const agg = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1793
|
+
for (const m of this.topicMetrics.values()) {
|
|
1794
|
+
agg.processedCount += m.processedCount;
|
|
1795
|
+
agg.retryCount += m.retryCount;
|
|
1796
|
+
agg.dlqCount += m.dlqCount;
|
|
1797
|
+
agg.dedupCount += m.dedupCount;
|
|
1798
|
+
}
|
|
1799
|
+
return agg;
|
|
1800
|
+
}
|
|
1801
|
+
resetMetrics(topic2) {
|
|
1802
|
+
if (topic2 !== void 0) {
|
|
1803
|
+
this.topicMetrics.delete(topic2);
|
|
1804
|
+
return;
|
|
1805
|
+
}
|
|
1806
|
+
this.topicMetrics.clear();
|
|
1807
|
+
}
|
|
1808
|
+
};
|
|
1809
|
+
|
|
1810
|
+
// src/client/kafka.client/infra/inflight-tracker.ts
|
|
1811
|
+
var InFlightTracker = class {
|
|
1812
|
+
constructor(warn) {
|
|
1813
|
+
this.warn = warn;
|
|
1814
|
+
}
|
|
1815
|
+
inFlightTotal = 0;
|
|
1816
|
+
drainResolvers = [];
|
|
1817
|
+
track(fn) {
|
|
1818
|
+
this.inFlightTotal++;
|
|
1819
|
+
return fn().finally(() => {
|
|
1820
|
+
this.inFlightTotal--;
|
|
1821
|
+
if (this.inFlightTotal === 0) this.drainResolvers.splice(0).forEach((r) => r());
|
|
1822
|
+
});
|
|
1823
|
+
}
|
|
1824
|
+
waitForDrain(timeoutMs) {
|
|
1825
|
+
if (this.inFlightTotal === 0) return Promise.resolve();
|
|
1826
|
+
return new Promise((resolve) => {
|
|
1827
|
+
let handle;
|
|
1828
|
+
const onDrain = () => {
|
|
1829
|
+
clearTimeout(handle);
|
|
1830
|
+
resolve();
|
|
1831
|
+
};
|
|
1832
|
+
this.drainResolvers.push(onDrain);
|
|
1833
|
+
handle = setTimeout(() => {
|
|
1834
|
+
const idx = this.drainResolvers.indexOf(onDrain);
|
|
1835
|
+
if (idx !== -1) this.drainResolvers.splice(idx, 1);
|
|
1836
|
+
this.warn(
|
|
1837
|
+
`Drain timed out after ${timeoutMs}ms \u2014 ${this.inFlightTotal} handler(s) still in flight`
|
|
1838
|
+
);
|
|
1839
|
+
resolve();
|
|
1840
|
+
}, timeoutMs);
|
|
1841
|
+
});
|
|
1842
|
+
}
|
|
1843
|
+
};
|
|
1844
|
+
|
|
1845
|
+
// src/client/kafka.client/index.ts
|
|
1846
|
+
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = import_kafka_javascript2.KafkaJS;
|
|
1847
|
+
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1294
1848
|
var KafkaClient = class _KafkaClient {
|
|
1295
1849
|
kafka;
|
|
1296
1850
|
producer;
|
|
@@ -1299,7 +1853,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1299
1853
|
/** Maps transactionalId → Producer for each active retry level consumer. */
|
|
1300
1854
|
retryTxProducers = /* @__PURE__ */ new Map();
|
|
1301
1855
|
consumers = /* @__PURE__ */ new Map();
|
|
1302
|
-
admin;
|
|
1303
1856
|
logger;
|
|
1304
1857
|
autoCreateTopicsEnabled;
|
|
1305
1858
|
strictSchemasEnabled;
|
|
@@ -1319,20 +1872,26 @@ var KafkaClient = class _KafkaClient {
|
|
|
1319
1872
|
onRebalance;
|
|
1320
1873
|
/** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
|
|
1321
1874
|
txId;
|
|
1322
|
-
/** Per-topic event counters, lazily created on first event. Aggregated by `getMetrics()`. */
|
|
1323
|
-
_topicMetrics = /* @__PURE__ */ new Map();
|
|
1324
1875
|
/** Monotonically increasing Lamport clock stamped on every outgoing message. */
|
|
1325
1876
|
_lamportClock = 0;
|
|
1326
1877
|
/** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
|
|
1327
1878
|
dedupStates = /* @__PURE__ */ new Map();
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
isAdminConnected = false;
|
|
1333
|
-
inFlightTotal = 0;
|
|
1334
|
-
drainResolvers = [];
|
|
1879
|
+
circuitBreaker;
|
|
1880
|
+
adminOps;
|
|
1881
|
+
metrics;
|
|
1882
|
+
inFlight;
|
|
1335
1883
|
clientId;
|
|
1884
|
+
_producerOpsDeps;
|
|
1885
|
+
_consumerOpsDeps;
|
|
1886
|
+
_retryTopicDeps;
|
|
1887
|
+
/** DLQ header keys added by the pipeline — stripped before re-publishing. */
|
|
1888
|
+
static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
1889
|
+
"x-dlq-original-topic",
|
|
1890
|
+
"x-dlq-failed-at",
|
|
1891
|
+
"x-dlq-error-message",
|
|
1892
|
+
"x-dlq-error-stack",
|
|
1893
|
+
"x-dlq-attempt-count"
|
|
1894
|
+
]);
|
|
1336
1895
|
constructor(clientId, groupId, brokers, options) {
|
|
1337
1896
|
this.clientId = clientId;
|
|
1338
1897
|
this.defaultGroupId = groupId;
|
|
@@ -1357,12 +1916,41 @@ var KafkaClient = class _KafkaClient {
|
|
|
1357
1916
|
logLevel: KafkaLogLevel.ERROR
|
|
1358
1917
|
}
|
|
1359
1918
|
});
|
|
1360
|
-
this.producer = this.kafka.producer({
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1919
|
+
this.producer = this.kafka.producer({ kafkaJS: { acks: -1 } });
|
|
1920
|
+
this.adminOps = new AdminOps({
|
|
1921
|
+
admin: this.kafka.admin(),
|
|
1922
|
+
logger: this.logger,
|
|
1923
|
+
runningConsumers: this.runningConsumers,
|
|
1924
|
+
defaultGroupId: this.defaultGroupId,
|
|
1925
|
+
clientId: this.clientId
|
|
1926
|
+
});
|
|
1927
|
+
this.circuitBreaker = new CircuitBreakerManager({
|
|
1928
|
+
pauseConsumer: (gid, assignments) => this.pauseConsumer(gid, assignments),
|
|
1929
|
+
resumeConsumer: (gid, assignments) => this.resumeConsumer(gid, assignments),
|
|
1930
|
+
logger: this.logger,
|
|
1931
|
+
instrumentation: this.instrumentation
|
|
1932
|
+
});
|
|
1933
|
+
this.metrics = new MetricsManager({
|
|
1934
|
+
instrumentation: this.instrumentation,
|
|
1935
|
+
onCircuitFailure: (envelope, gid) => this.circuitBreaker.onFailure(envelope, gid),
|
|
1936
|
+
onCircuitSuccess: (envelope, gid) => this.circuitBreaker.onSuccess(envelope, gid)
|
|
1364
1937
|
});
|
|
1365
|
-
this.
|
|
1938
|
+
this.inFlight = new InFlightTracker((msg) => this.logger.warn(msg));
|
|
1939
|
+
this._producerOpsDeps = {
|
|
1940
|
+
schemaRegistry: this.schemaRegistry,
|
|
1941
|
+
strictSchemasEnabled: this.strictSchemasEnabled,
|
|
1942
|
+
instrumentation: this.instrumentation,
|
|
1943
|
+
logger: this.logger,
|
|
1944
|
+
nextLamportClock: () => ++this._lamportClock
|
|
1945
|
+
};
|
|
1946
|
+
this._consumerOpsDeps = {
|
|
1947
|
+
consumers: this.consumers,
|
|
1948
|
+
consumerCreationOptions: this.consumerCreationOptions,
|
|
1949
|
+
kafka: this.kafka,
|
|
1950
|
+
onRebalance: this.onRebalance,
|
|
1951
|
+
logger: this.logger
|
|
1952
|
+
};
|
|
1953
|
+
this._retryTopicDeps = this.buildRetryTopicDeps();
|
|
1366
1954
|
}
|
|
1367
1955
|
async sendMessage(topicOrDesc, message, options = {}) {
|
|
1368
1956
|
const payload = await this.preparePayload(
|
|
@@ -1380,7 +1968,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1380
1968
|
options.compression
|
|
1381
1969
|
);
|
|
1382
1970
|
await this.producer.send(payload);
|
|
1383
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1971
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1384
1972
|
}
|
|
1385
1973
|
/**
|
|
1386
1974
|
* Send a null-value (tombstone) message. Used with log-compacted topics to signal
|
|
@@ -1395,26 +1983,15 @@ var KafkaClient = class _KafkaClient {
|
|
|
1395
1983
|
*/
|
|
1396
1984
|
async sendTombstone(topic2, key, headers) {
|
|
1397
1985
|
const hdrs = { ...headers };
|
|
1398
|
-
for (const inst of this.instrumentation)
|
|
1399
|
-
inst.beforeSend?.(topic2, hdrs);
|
|
1400
|
-
}
|
|
1986
|
+
for (const inst of this.instrumentation) inst.beforeSend?.(topic2, hdrs);
|
|
1401
1987
|
await this.ensureTopic(topic2);
|
|
1402
|
-
await this.producer.send({
|
|
1403
|
-
|
|
1404
|
-
messages: [{ value: null, key, headers: hdrs }]
|
|
1405
|
-
});
|
|
1406
|
-
for (const inst of this.instrumentation) {
|
|
1407
|
-
inst.afterSend?.(topic2);
|
|
1408
|
-
}
|
|
1988
|
+
await this.producer.send({ topic: topic2, messages: [{ value: null, key, headers: hdrs }] });
|
|
1989
|
+
for (const inst of this.instrumentation) inst.afterSend?.(topic2);
|
|
1409
1990
|
}
|
|
1410
1991
|
async sendBatch(topicOrDesc, messages, options) {
|
|
1411
|
-
const payload = await this.preparePayload(
|
|
1412
|
-
topicOrDesc,
|
|
1413
|
-
messages,
|
|
1414
|
-
options?.compression
|
|
1415
|
-
);
|
|
1992
|
+
const payload = await this.preparePayload(topicOrDesc, messages, options?.compression);
|
|
1416
1993
|
await this.producer.send(payload);
|
|
1417
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1994
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1418
1995
|
}
|
|
1419
1996
|
/** Execute multiple sends atomically. Commits on success, aborts on error. */
|
|
1420
1997
|
async transaction(fn) {
|
|
@@ -1426,12 +2003,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1426
2003
|
}
|
|
1427
2004
|
const initPromise = (async () => {
|
|
1428
2005
|
const p = this.kafka.producer({
|
|
1429
|
-
kafkaJS: {
|
|
1430
|
-
acks: -1,
|
|
1431
|
-
idempotent: true,
|
|
1432
|
-
transactionalId: this.txId,
|
|
1433
|
-
maxInFlightRequests: 1
|
|
1434
|
-
}
|
|
2006
|
+
kafkaJS: { acks: -1, idempotent: true, transactionalId: this.txId, maxInFlightRequests: 1 }
|
|
1435
2007
|
});
|
|
1436
2008
|
await p.connect();
|
|
1437
2009
|
_activeTransactionalIds.add(this.txId);
|
|
@@ -1458,19 +2030,12 @@ var KafkaClient = class _KafkaClient {
|
|
|
1458
2030
|
}
|
|
1459
2031
|
]);
|
|
1460
2032
|
await tx.send(payload);
|
|
1461
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
2033
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1462
2034
|
},
|
|
1463
|
-
/**
|
|
1464
|
-
* Send multiple messages in a single call to the topic.
|
|
1465
|
-
* All messages in the batch will be sent atomically.
|
|
1466
|
-
* If any message fails to send, the entire batch will be aborted.
|
|
1467
|
-
* @param topicOrDesc - topic name or TopicDescriptor
|
|
1468
|
-
* @param messages - array of messages to send with optional key, headers, correlationId, schemaVersion, and eventId
|
|
1469
|
-
*/
|
|
1470
2035
|
sendBatch: async (topicOrDesc, messages) => {
|
|
1471
2036
|
const payload = await this.preparePayload(topicOrDesc, messages);
|
|
1472
2037
|
await tx.send(payload);
|
|
1473
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
2038
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1474
2039
|
}
|
|
1475
2040
|
};
|
|
1476
2041
|
await fn(ctx);
|
|
@@ -1479,10 +2044,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1479
2044
|
try {
|
|
1480
2045
|
await tx.abort();
|
|
1481
2046
|
} catch (abortError) {
|
|
1482
|
-
this.logger.error(
|
|
1483
|
-
"Failed to abort transaction:",
|
|
1484
|
-
toError(abortError).message
|
|
1485
|
-
);
|
|
2047
|
+
this.logger.error("Failed to abort transaction:", toError(abortError).message);
|
|
1486
2048
|
}
|
|
1487
2049
|
throw error;
|
|
1488
2050
|
}
|
|
@@ -1504,35 +2066,14 @@ var KafkaClient = class _KafkaClient {
|
|
|
1504
2066
|
this.logger.log("Producer disconnected");
|
|
1505
2067
|
}
|
|
1506
2068
|
async startConsumer(topics, handleMessage, options = {}) {
|
|
1507
|
-
|
|
1508
|
-
throw new Error(
|
|
1509
|
-
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1510
|
-
);
|
|
1511
|
-
}
|
|
1512
|
-
const hasRegexTopics = topics.some((t) => t instanceof RegExp);
|
|
1513
|
-
if (options.retryTopics && hasRegexTopics) {
|
|
1514
|
-
throw new Error(
|
|
1515
|
-
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
1516
|
-
);
|
|
1517
|
-
}
|
|
2069
|
+
this.validateTopicConsumerOpts(topics, options);
|
|
1518
2070
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1519
2071
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", setupOptions);
|
|
1520
|
-
if (options.circuitBreaker)
|
|
1521
|
-
this.circuitConfigs.set(gid, options.circuitBreaker);
|
|
2072
|
+
if (options.circuitBreaker) this.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
1522
2073
|
const deps = this.messageDepsFor(gid);
|
|
1523
|
-
const
|
|
1524
|
-
const deduplication = this.resolveDeduplicationContext(
|
|
1525
|
-
gid,
|
|
1526
|
-
options.deduplication
|
|
1527
|
-
);
|
|
1528
|
-
let eosMainContext;
|
|
1529
|
-
if (options.retryTopics && retry) {
|
|
1530
|
-
const mainTxId = `${gid}-main-tx`;
|
|
1531
|
-
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1532
|
-
eosMainContext = { txProducer, consumer };
|
|
1533
|
-
}
|
|
2074
|
+
const eosMainContext = await this.makeEosMainContext(gid, consumer, options);
|
|
1534
2075
|
await consumer.run({
|
|
1535
|
-
eachMessage: (payload) => this.
|
|
2076
|
+
eachMessage: (payload) => this.inFlight.track(
|
|
1536
2077
|
() => handleEachMessage(
|
|
1537
2078
|
payload,
|
|
1538
2079
|
{
|
|
@@ -1542,9 +2083,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1542
2083
|
dlq,
|
|
1543
2084
|
retry,
|
|
1544
2085
|
retryTopics: options.retryTopics,
|
|
1545
|
-
timeoutMs,
|
|
2086
|
+
timeoutMs: options.handlerTimeoutMs,
|
|
1546
2087
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1547
|
-
deduplication,
|
|
2088
|
+
deduplication: this.resolveDeduplicationContext(gid, options.deduplication),
|
|
1548
2089
|
messageTtlMs: options.messageTtlMs,
|
|
1549
2090
|
onTtlExpired: options.onTtlExpired,
|
|
1550
2091
|
eosMainContext
|
|
@@ -1555,70 +2096,24 @@ var KafkaClient = class _KafkaClient {
|
|
|
1555
2096
|
});
|
|
1556
2097
|
this.runningConsumers.set(gid, "eachMessage");
|
|
1557
2098
|
if (options.retryTopics && retry) {
|
|
1558
|
-
|
|
1559
|
-
await this.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
1560
|
-
}
|
|
1561
|
-
const companions = await startRetryTopicConsumers(
|
|
1562
|
-
topicNames,
|
|
1563
|
-
gid,
|
|
1564
|
-
handleMessage,
|
|
1565
|
-
retry,
|
|
1566
|
-
dlq,
|
|
1567
|
-
interceptors,
|
|
1568
|
-
schemaMap,
|
|
1569
|
-
this.retryTopicDeps,
|
|
1570
|
-
options.retryTopicAssignmentTimeoutMs
|
|
1571
|
-
);
|
|
1572
|
-
this.companionGroupIds.set(gid, companions);
|
|
2099
|
+
await this.launchRetryChain(gid, topicNames, handleMessage, retry, dlq, interceptors, schemaMap, options.retryTopicAssignmentTimeoutMs);
|
|
1573
2100
|
}
|
|
1574
2101
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
1575
2102
|
}
|
|
1576
2103
|
async startBatchConsumer(topics, handleBatch, options = {}) {
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1580
|
-
);
|
|
1581
|
-
}
|
|
1582
|
-
const hasRegexTopics = topics.some((t) => t instanceof RegExp);
|
|
1583
|
-
if (options.retryTopics && hasRegexTopics) {
|
|
1584
|
-
throw new Error(
|
|
1585
|
-
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
1586
|
-
);
|
|
1587
|
-
}
|
|
1588
|
-
if (options.retryTopics) {
|
|
1589
|
-
} else if (options.autoCommit !== false) {
|
|
2104
|
+
this.validateTopicConsumerOpts(topics, options);
|
|
2105
|
+
if (!options.retryTopics && options.autoCommit !== false) {
|
|
1590
2106
|
this.logger.debug?.(
|
|
1591
2107
|
`startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
|
|
1592
2108
|
);
|
|
1593
2109
|
}
|
|
1594
2110
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1595
2111
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", setupOptions);
|
|
1596
|
-
if (options.circuitBreaker)
|
|
1597
|
-
this.circuitConfigs.set(gid, options.circuitBreaker);
|
|
2112
|
+
if (options.circuitBreaker) this.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
1598
2113
|
const deps = this.messageDepsFor(gid);
|
|
1599
|
-
const
|
|
1600
|
-
const deduplication = this.resolveDeduplicationContext(
|
|
1601
|
-
gid,
|
|
1602
|
-
options.deduplication
|
|
1603
|
-
);
|
|
1604
|
-
let eosMainContext;
|
|
1605
|
-
if (options.retryTopics && retry) {
|
|
1606
|
-
const mainTxId = `${gid}-main-tx`;
|
|
1607
|
-
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1608
|
-
eosMainContext = { txProducer, consumer };
|
|
1609
|
-
}
|
|
2114
|
+
const eosMainContext = await this.makeEosMainContext(gid, consumer, options);
|
|
1610
2115
|
await consumer.run({
|
|
1611
|
-
|
|
1612
|
-
* eachBatch: called by the consumer for each batch of messages.
|
|
1613
|
-
* Called with the `payload` argument, which is an object containing the
|
|
1614
|
-
* batch of messages and a `BatchMeta` object with offset management controls.
|
|
1615
|
-
*
|
|
1616
|
-
* The function is wrapped with `trackInFlight` and `handleEachBatch` to provide
|
|
1617
|
-
* error handling and offset management.
|
|
1618
|
-
*
|
|
1619
|
-
* @param payload - an object containing the batch of messages and a `BatchMeta` object.
|
|
1620
|
-
*/
|
|
1621
|
-
eachBatch: (payload) => this.trackInFlight(
|
|
2116
|
+
eachBatch: (payload) => this.inFlight.track(
|
|
1622
2117
|
() => handleEachBatch(
|
|
1623
2118
|
payload,
|
|
1624
2119
|
{
|
|
@@ -1628,9 +2123,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1628
2123
|
dlq,
|
|
1629
2124
|
retry,
|
|
1630
2125
|
retryTopics: options.retryTopics,
|
|
1631
|
-
timeoutMs,
|
|
2126
|
+
timeoutMs: options.handlerTimeoutMs,
|
|
1632
2127
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1633
|
-
deduplication,
|
|
2128
|
+
deduplication: this.resolveDeduplicationContext(gid, options.deduplication),
|
|
1634
2129
|
messageTtlMs: options.messageTtlMs,
|
|
1635
2130
|
onTtlExpired: options.onTtlExpired,
|
|
1636
2131
|
eosMainContext
|
|
@@ -1641,9 +2136,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1641
2136
|
});
|
|
1642
2137
|
this.runningConsumers.set(gid, "eachBatch");
|
|
1643
2138
|
if (options.retryTopics && retry) {
|
|
1644
|
-
if (!this.autoCreateTopicsEnabled) {
|
|
1645
|
-
await this.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
1646
|
-
}
|
|
1647
2139
|
const handleMessageForRetry = (env) => handleBatch([env], {
|
|
1648
2140
|
partition: env.partition,
|
|
1649
2141
|
highWatermark: null,
|
|
@@ -1654,18 +2146,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1654
2146
|
commitOffsetsIfNecessary: async () => {
|
|
1655
2147
|
}
|
|
1656
2148
|
});
|
|
1657
|
-
|
|
1658
|
-
topicNames,
|
|
1659
|
-
gid,
|
|
1660
|
-
handleMessageForRetry,
|
|
1661
|
-
retry,
|
|
1662
|
-
dlq,
|
|
1663
|
-
interceptors,
|
|
1664
|
-
schemaMap,
|
|
1665
|
-
this.retryTopicDeps,
|
|
1666
|
-
options.retryTopicAssignmentTimeoutMs
|
|
1667
|
-
);
|
|
1668
|
-
this.companionGroupIds.set(gid, companions);
|
|
2149
|
+
await this.launchRetryChain(gid, topicNames, handleMessageForRetry, retry, dlq, interceptors, schemaMap, options.retryTopicAssignmentTimeoutMs);
|
|
1669
2150
|
}
|
|
1670
2151
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
1671
2152
|
}
|
|
@@ -1724,38 +2205,20 @@ var KafkaClient = class _KafkaClient {
|
|
|
1724
2205
|
if (groupId !== void 0) {
|
|
1725
2206
|
const consumer = this.consumers.get(groupId);
|
|
1726
2207
|
if (!consumer) {
|
|
1727
|
-
this.logger.warn(
|
|
1728
|
-
`stopConsumer: no active consumer for group "${groupId}"`
|
|
1729
|
-
);
|
|
2208
|
+
this.logger.warn(`stopConsumer: no active consumer for group "${groupId}"`);
|
|
1730
2209
|
return;
|
|
1731
2210
|
}
|
|
1732
|
-
await consumer.disconnect().catch(
|
|
1733
|
-
(e) => this.logger.warn(
|
|
1734
|
-
`Error disconnecting consumer "${groupId}":`,
|
|
1735
|
-
toError(e).message
|
|
1736
|
-
)
|
|
1737
|
-
);
|
|
2211
|
+
await consumer.disconnect().catch((e) => this.logger.warn(`Error disconnecting consumer "${groupId}":`, toError(e).message));
|
|
1738
2212
|
this.consumers.delete(groupId);
|
|
1739
2213
|
this.runningConsumers.delete(groupId);
|
|
1740
2214
|
this.consumerCreationOptions.delete(groupId);
|
|
1741
2215
|
this.dedupStates.delete(groupId);
|
|
1742
|
-
|
|
1743
|
-
if (key.startsWith(`${groupId}:`)) {
|
|
1744
|
-
clearTimeout(this.circuitStates.get(key).timer);
|
|
1745
|
-
this.circuitStates.delete(key);
|
|
1746
|
-
}
|
|
1747
|
-
}
|
|
1748
|
-
this.circuitConfigs.delete(groupId);
|
|
2216
|
+
this.circuitBreaker.removeGroup(groupId);
|
|
1749
2217
|
this.logger.log(`Consumer disconnected: group "${groupId}"`);
|
|
1750
2218
|
const mainTxId = `${groupId}-main-tx`;
|
|
1751
2219
|
const mainTxProducer = this.retryTxProducers.get(mainTxId);
|
|
1752
2220
|
if (mainTxProducer) {
|
|
1753
|
-
await mainTxProducer.disconnect().catch(
|
|
1754
|
-
(e) => this.logger.warn(
|
|
1755
|
-
`Error disconnecting main tx producer "${mainTxId}":`,
|
|
1756
|
-
toError(e).message
|
|
1757
|
-
)
|
|
1758
|
-
);
|
|
2221
|
+
await mainTxProducer.disconnect().catch((e) => this.logger.warn(`Error disconnecting main tx producer "${mainTxId}":`, toError(e).message));
|
|
1759
2222
|
_activeTransactionalIds.delete(mainTxId);
|
|
1760
2223
|
this.retryTxProducers.delete(mainTxId);
|
|
1761
2224
|
}
|
|
@@ -1763,12 +2226,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1763
2226
|
for (const cGroupId of companions) {
|
|
1764
2227
|
const cConsumer = this.consumers.get(cGroupId);
|
|
1765
2228
|
if (cConsumer) {
|
|
1766
|
-
await cConsumer.disconnect().catch(
|
|
1767
|
-
(e) => this.logger.warn(
|
|
1768
|
-
`Error disconnecting retry consumer "${cGroupId}":`,
|
|
1769
|
-
toError(e).message
|
|
1770
|
-
)
|
|
1771
|
-
);
|
|
2229
|
+
await cConsumer.disconnect().catch((e) => this.logger.warn(`Error disconnecting retry consumer "${cGroupId}":`, toError(e).message));
|
|
1772
2230
|
this.consumers.delete(cGroupId);
|
|
1773
2231
|
this.runningConsumers.delete(cGroupId);
|
|
1774
2232
|
this.consumerCreationOptions.delete(cGroupId);
|
|
@@ -1777,12 +2235,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1777
2235
|
const txId = `${cGroupId}-tx`;
|
|
1778
2236
|
const txProducer = this.retryTxProducers.get(txId);
|
|
1779
2237
|
if (txProducer) {
|
|
1780
|
-
await txProducer.disconnect().catch(
|
|
1781
|
-
(e) => this.logger.warn(
|
|
1782
|
-
`Error disconnecting retry tx producer "${txId}":`,
|
|
1783
|
-
toError(e).message
|
|
1784
|
-
)
|
|
1785
|
-
);
|
|
2238
|
+
await txProducer.disconnect().catch((e) => this.logger.warn(`Error disconnecting retry tx producer "${txId}":`, toError(e).message));
|
|
1786
2239
|
_activeTransactionalIds.delete(txId);
|
|
1787
2240
|
this.retryTxProducers.delete(txId);
|
|
1788
2241
|
}
|
|
@@ -1790,14 +2243,10 @@ var KafkaClient = class _KafkaClient {
|
|
|
1790
2243
|
this.companionGroupIds.delete(groupId);
|
|
1791
2244
|
} else {
|
|
1792
2245
|
const tasks = [
|
|
1793
|
-
...Array.from(this.consumers.values()).map(
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
)
|
|
1797
|
-
...Array.from(this.retryTxProducers.values()).map(
|
|
1798
|
-
(p) => p.disconnect().catch(() => {
|
|
1799
|
-
})
|
|
1800
|
-
)
|
|
2246
|
+
...Array.from(this.consumers.values()).map((c) => c.disconnect().catch(() => {
|
|
2247
|
+
})),
|
|
2248
|
+
...Array.from(this.retryTxProducers.values()).map((p) => p.disconnect().catch(() => {
|
|
2249
|
+
}))
|
|
1801
2250
|
];
|
|
1802
2251
|
await Promise.allSettled(tasks);
|
|
1803
2252
|
this.consumers.clear();
|
|
@@ -1806,10 +2255,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1806
2255
|
this.companionGroupIds.clear();
|
|
1807
2256
|
this.retryTxProducers.clear();
|
|
1808
2257
|
this.dedupStates.clear();
|
|
1809
|
-
|
|
1810
|
-
clearTimeout(state.timer);
|
|
1811
|
-
this.circuitStates.clear();
|
|
1812
|
-
this.circuitConfigs.clear();
|
|
2258
|
+
this.circuitBreaker.clear();
|
|
1813
2259
|
this.logger.log("All consumers disconnected");
|
|
1814
2260
|
}
|
|
1815
2261
|
}
|
|
@@ -1827,9 +2273,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1827
2273
|
return;
|
|
1828
2274
|
}
|
|
1829
2275
|
consumer.pause(
|
|
1830
|
-
assignments.flatMap(
|
|
1831
|
-
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1832
|
-
)
|
|
2276
|
+
assignments.flatMap(({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] })))
|
|
1833
2277
|
);
|
|
1834
2278
|
}
|
|
1835
2279
|
/**
|
|
@@ -1846,9 +2290,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1846
2290
|
return;
|
|
1847
2291
|
}
|
|
1848
2292
|
consumer.resume(
|
|
1849
|
-
assignments.flatMap(
|
|
1850
|
-
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1851
|
-
)
|
|
2293
|
+
assignments.flatMap(({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] })))
|
|
1852
2294
|
);
|
|
1853
2295
|
}
|
|
1854
2296
|
/** Pause all assigned partitions of a topic for a consumer group (used for queue backpressure). */
|
|
@@ -1869,14 +2311,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1869
2311
|
if (partitions.length > 0)
|
|
1870
2312
|
consumer.resume(partitions.map((p) => ({ topic: topic2, partitions: [p] })));
|
|
1871
2313
|
}
|
|
1872
|
-
/** DLQ header keys added by `sendToDlq` — stripped before re-publishing. */
|
|
1873
|
-
static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
1874
|
-
"x-dlq-original-topic",
|
|
1875
|
-
"x-dlq-failed-at",
|
|
1876
|
-
"x-dlq-error-message",
|
|
1877
|
-
"x-dlq-error-stack",
|
|
1878
|
-
"x-dlq-attempt-count"
|
|
1879
|
-
]);
|
|
1880
2314
|
/**
|
|
1881
2315
|
* Re-publish messages from a dead letter queue back to the original topic.
|
|
1882
2316
|
*
|
|
@@ -1889,106 +2323,27 @@ var KafkaClient = class _KafkaClient {
|
|
|
1889
2323
|
* @returns { replayed: number; skipped: number } - counts of re-published vs skipped messages
|
|
1890
2324
|
*/
|
|
1891
2325
|
async replayDlq(topic2, options = {}) {
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
}
|
|
1902
|
-
const highWatermarks = new Map(
|
|
1903
|
-
activePartitions.map(({ partition, high }) => [
|
|
1904
|
-
partition,
|
|
1905
|
-
parseInt(high, 10)
|
|
1906
|
-
])
|
|
1907
|
-
);
|
|
1908
|
-
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1909
|
-
let replayed = 0;
|
|
1910
|
-
let skipped = 0;
|
|
1911
|
-
const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
|
|
1912
|
-
await new Promise((resolve, reject) => {
|
|
1913
|
-
const consumer = getOrCreateConsumer(
|
|
1914
|
-
tempGroupId,
|
|
1915
|
-
true,
|
|
1916
|
-
true,
|
|
1917
|
-
this.consumerOpsDeps
|
|
1918
|
-
);
|
|
1919
|
-
const cleanup = () => {
|
|
2326
|
+
await this.adminOps.ensureConnected();
|
|
2327
|
+
return replayDlqTopic(topic2, options, {
|
|
2328
|
+
logger: this.logger,
|
|
2329
|
+
fetchTopicOffsets: (t) => this.adminOps.admin.fetchTopicOffsets(t),
|
|
2330
|
+
send: async (t, messages) => {
|
|
2331
|
+
await this.producer.send({ topic: t, messages });
|
|
2332
|
+
},
|
|
2333
|
+
createConsumer: (gid) => getOrCreateConsumer(gid, true, true, this._consumerOpsDeps),
|
|
2334
|
+
cleanupConsumer: (consumer, gid) => {
|
|
1920
2335
|
consumer.disconnect().catch(() => {
|
|
1921
2336
|
}).finally(() => {
|
|
1922
|
-
this.consumers.delete(
|
|
1923
|
-
this.runningConsumers.delete(
|
|
1924
|
-
this.consumerCreationOptions.delete(
|
|
2337
|
+
this.consumers.delete(gid);
|
|
2338
|
+
this.runningConsumers.delete(gid);
|
|
2339
|
+
this.consumerCreationOptions.delete(gid);
|
|
1925
2340
|
});
|
|
1926
|
-
}
|
|
1927
|
-
|
|
1928
|
-
() => consumer.run({
|
|
1929
|
-
eachMessage: async ({ partition, message }) => {
|
|
1930
|
-
if (!message.value) return;
|
|
1931
|
-
const offset = parseInt(message.offset, 10);
|
|
1932
|
-
processedOffsets.set(partition, offset);
|
|
1933
|
-
const headers = decodeHeaders(message.headers);
|
|
1934
|
-
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1935
|
-
const originalHeaders = Object.fromEntries(
|
|
1936
|
-
Object.entries(headers).filter(
|
|
1937
|
-
([k]) => !_KafkaClient.DLQ_HEADER_KEYS.has(k)
|
|
1938
|
-
)
|
|
1939
|
-
);
|
|
1940
|
-
const value = message.value.toString();
|
|
1941
|
-
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1942
|
-
if (!targetTopic || !shouldProcess) {
|
|
1943
|
-
skipped++;
|
|
1944
|
-
} else if (options.dryRun) {
|
|
1945
|
-
this.logger.log(
|
|
1946
|
-
`[DLQ replay dry-run] Would replay to "${targetTopic}"`
|
|
1947
|
-
);
|
|
1948
|
-
replayed++;
|
|
1949
|
-
} else {
|
|
1950
|
-
await this.producer.send({
|
|
1951
|
-
topic: targetTopic,
|
|
1952
|
-
messages: [{ value, headers: originalHeaders }]
|
|
1953
|
-
});
|
|
1954
|
-
replayed++;
|
|
1955
|
-
}
|
|
1956
|
-
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1957
|
-
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1958
|
-
);
|
|
1959
|
-
if (allDone) {
|
|
1960
|
-
cleanup();
|
|
1961
|
-
resolve();
|
|
1962
|
-
}
|
|
1963
|
-
}
|
|
1964
|
-
})
|
|
1965
|
-
).catch((err) => {
|
|
1966
|
-
cleanup();
|
|
1967
|
-
reject(err);
|
|
1968
|
-
});
|
|
2341
|
+
},
|
|
2342
|
+
dlqHeaderKeys: _KafkaClient.DLQ_HEADER_KEYS
|
|
1969
2343
|
});
|
|
1970
|
-
this.logger.log(
|
|
1971
|
-
`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`
|
|
1972
|
-
);
|
|
1973
|
-
return { replayed, skipped };
|
|
1974
2344
|
}
|
|
1975
2345
|
async resetOffsets(groupId, topic2, position) {
|
|
1976
|
-
|
|
1977
|
-
if (this.runningConsumers.has(gid)) {
|
|
1978
|
-
throw new Error(
|
|
1979
|
-
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
1980
|
-
);
|
|
1981
|
-
}
|
|
1982
|
-
await this.ensureAdminConnected();
|
|
1983
|
-
const partitionOffsets = await this.admin.fetchTopicOffsets(topic2);
|
|
1984
|
-
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
1985
|
-
partition,
|
|
1986
|
-
offset: position === "earliest" ? low : high
|
|
1987
|
-
}));
|
|
1988
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1989
|
-
this.logger.log(
|
|
1990
|
-
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1991
|
-
);
|
|
2346
|
+
return this.adminOps.resetOffsets(groupId, topic2, position);
|
|
1992
2347
|
}
|
|
1993
2348
|
/**
|
|
1994
2349
|
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
@@ -1996,25 +2351,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1996
2351
|
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1997
2352
|
*/
|
|
1998
2353
|
async seekToOffset(groupId, assignments) {
|
|
1999
|
-
|
|
2000
|
-
if (this.runningConsumers.has(gid)) {
|
|
2001
|
-
throw new Error(
|
|
2002
|
-
`seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
2003
|
-
);
|
|
2004
|
-
}
|
|
2005
|
-
await this.ensureAdminConnected();
|
|
2006
|
-
const byTopic = /* @__PURE__ */ new Map();
|
|
2007
|
-
for (const { topic: topic2, partition, offset } of assignments) {
|
|
2008
|
-
const list = byTopic.get(topic2) ?? [];
|
|
2009
|
-
list.push({ partition, offset });
|
|
2010
|
-
byTopic.set(topic2, list);
|
|
2011
|
-
}
|
|
2012
|
-
for (const [topic2, partitions] of byTopic) {
|
|
2013
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
2014
|
-
this.logger.log(
|
|
2015
|
-
`Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
|
|
2016
|
-
);
|
|
2017
|
-
}
|
|
2354
|
+
return this.adminOps.seekToOffset(groupId, assignments);
|
|
2018
2355
|
}
|
|
2019
2356
|
/**
|
|
2020
2357
|
* Seek specific topic-partition pairs to the offset nearest to a given timestamp
|
|
@@ -2025,37 +2362,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
2025
2362
|
* future timestamp), the partition falls back to `-1` (end of topic — new messages only).
|
|
2026
2363
|
*/
|
|
2027
2364
|
async seekToTimestamp(groupId, assignments) {
|
|
2028
|
-
|
|
2029
|
-
if (this.runningConsumers.has(gid)) {
|
|
2030
|
-
throw new Error(
|
|
2031
|
-
`seekToTimestamp: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
2032
|
-
);
|
|
2033
|
-
}
|
|
2034
|
-
await this.ensureAdminConnected();
|
|
2035
|
-
const byTopic = /* @__PURE__ */ new Map();
|
|
2036
|
-
for (const { topic: topic2, partition, timestamp } of assignments) {
|
|
2037
|
-
const list = byTopic.get(topic2) ?? [];
|
|
2038
|
-
list.push({ partition, timestamp });
|
|
2039
|
-
byTopic.set(topic2, list);
|
|
2040
|
-
}
|
|
2041
|
-
for (const [topic2, parts] of byTopic) {
|
|
2042
|
-
const offsets = await Promise.all(
|
|
2043
|
-
parts.map(async ({ partition, timestamp }) => {
|
|
2044
|
-
const results = await this.admin.fetchTopicOffsetsByTime(
|
|
2045
|
-
topic2,
|
|
2046
|
-
timestamp
|
|
2047
|
-
);
|
|
2048
|
-
const found = results.find(
|
|
2049
|
-
(r) => r.partition === partition
|
|
2050
|
-
);
|
|
2051
|
-
return { partition, offset: found?.offset ?? "-1" };
|
|
2052
|
-
})
|
|
2053
|
-
);
|
|
2054
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
|
|
2055
|
-
this.logger.log(
|
|
2056
|
-
`Offsets set by timestamp for group "${gid}" on "${topic2}": ${JSON.stringify(offsets)}`
|
|
2057
|
-
);
|
|
2058
|
-
}
|
|
2365
|
+
return this.adminOps.seekToTimestamp(groupId, assignments);
|
|
2059
2366
|
}
|
|
2060
2367
|
/**
|
|
2061
2368
|
* Returns the current circuit breaker state for a specific topic partition.
|
|
@@ -2069,14 +2376,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
2069
2376
|
* @returns `{ status, failures, windowSize }` snapshot for a given partition or `undefined` if no state exists.
|
|
2070
2377
|
*/
|
|
2071
2378
|
getCircuitState(topic2, partition, groupId) {
|
|
2072
|
-
|
|
2073
|
-
const state = this.circuitStates.get(`${gid}:${topic2}:${partition}`);
|
|
2074
|
-
if (!state) return void 0;
|
|
2075
|
-
return {
|
|
2076
|
-
status: state.status,
|
|
2077
|
-
failures: state.window.filter((v) => !v).length,
|
|
2078
|
-
windowSize: state.window.length
|
|
2079
|
-
};
|
|
2379
|
+
return this.circuitBreaker.getState(topic2, partition, groupId ?? this.defaultGroupId);
|
|
2080
2380
|
}
|
|
2081
2381
|
/**
|
|
2082
2382
|
* Query consumer group lag per partition.
|
|
@@ -2090,83 +2390,32 @@ var KafkaClient = class _KafkaClient {
|
|
|
2090
2390
|
* committed offset. Use `checkStatus()` to verify broker connectivity in that case.
|
|
2091
2391
|
*/
|
|
2092
2392
|
async getConsumerLag(groupId) {
|
|
2093
|
-
|
|
2094
|
-
await this.ensureAdminConnected();
|
|
2095
|
-
const committedByTopic = await this.admin.fetchOffsets({ groupId: gid });
|
|
2096
|
-
const brokerOffsetsAll = await Promise.all(
|
|
2097
|
-
committedByTopic.map(({ topic: topic2 }) => this.admin.fetchTopicOffsets(topic2))
|
|
2098
|
-
);
|
|
2099
|
-
const result = [];
|
|
2100
|
-
for (let i = 0; i < committedByTopic.length; i++) {
|
|
2101
|
-
const { topic: topic2, partitions } = committedByTopic[i];
|
|
2102
|
-
const brokerOffsets = brokerOffsetsAll[i];
|
|
2103
|
-
for (const { partition, offset } of partitions) {
|
|
2104
|
-
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
2105
|
-
if (!broker) continue;
|
|
2106
|
-
const committed = parseInt(offset, 10);
|
|
2107
|
-
const high = parseInt(broker.high, 10);
|
|
2108
|
-
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
2109
|
-
result.push({ topic: topic2, partition, lag });
|
|
2110
|
-
}
|
|
2111
|
-
}
|
|
2112
|
-
return result;
|
|
2393
|
+
return this.adminOps.getConsumerLag(groupId);
|
|
2113
2394
|
}
|
|
2114
2395
|
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
2115
2396
|
async checkStatus() {
|
|
2116
|
-
|
|
2117
|
-
await this.ensureAdminConnected();
|
|
2118
|
-
const topics = await this.admin.listTopics();
|
|
2119
|
-
return { status: "up", clientId: this.clientId, topics };
|
|
2120
|
-
} catch (error) {
|
|
2121
|
-
return {
|
|
2122
|
-
status: "down",
|
|
2123
|
-
clientId: this.clientId,
|
|
2124
|
-
error: error instanceof Error ? error.message : String(error)
|
|
2125
|
-
};
|
|
2126
|
-
}
|
|
2397
|
+
return this.adminOps.checkStatus();
|
|
2127
2398
|
}
|
|
2128
2399
|
/**
|
|
2129
2400
|
* List all consumer groups known to the broker.
|
|
2130
2401
|
* Useful for monitoring which groups are active and their current state.
|
|
2131
2402
|
*/
|
|
2132
2403
|
async listConsumerGroups() {
|
|
2133
|
-
|
|
2134
|
-
const result = await this.admin.listGroups();
|
|
2135
|
-
return result.groups.map((g) => ({
|
|
2136
|
-
groupId: g.groupId,
|
|
2137
|
-
state: g.state ?? "Unknown"
|
|
2138
|
-
}));
|
|
2404
|
+
return this.adminOps.listConsumerGroups();
|
|
2139
2405
|
}
|
|
2140
2406
|
/**
|
|
2141
2407
|
* Describe topics — returns partition layout, leader, replicas, and ISR.
|
|
2142
2408
|
* @param topics Topic names to describe. Omit to describe all topics.
|
|
2143
2409
|
*/
|
|
2144
2410
|
async describeTopics(topics) {
|
|
2145
|
-
|
|
2146
|
-
const result = await this.admin.fetchTopicMetadata(
|
|
2147
|
-
topics ? { topics } : void 0
|
|
2148
|
-
);
|
|
2149
|
-
return result.topics.map((t) => ({
|
|
2150
|
-
name: t.name,
|
|
2151
|
-
partitions: t.partitions.map((p) => ({
|
|
2152
|
-
partition: p.partitionId ?? p.partition,
|
|
2153
|
-
leader: p.leader,
|
|
2154
|
-
replicas: p.replicas.map(
|
|
2155
|
-
(r) => typeof r === "number" ? r : r.nodeId
|
|
2156
|
-
),
|
|
2157
|
-
isr: p.isr.map(
|
|
2158
|
-
(r) => typeof r === "number" ? r : r.nodeId
|
|
2159
|
-
)
|
|
2160
|
-
}))
|
|
2161
|
-
}));
|
|
2411
|
+
return this.adminOps.describeTopics(topics);
|
|
2162
2412
|
}
|
|
2163
2413
|
/**
|
|
2164
2414
|
* Delete records from a topic up to (but not including) the given offsets.
|
|
2165
2415
|
* All messages with offsets **before** the given offset are deleted.
|
|
2166
2416
|
*/
|
|
2167
2417
|
async deleteRecords(topic2, partitions) {
|
|
2168
|
-
|
|
2169
|
-
await this.admin.deleteTopicRecords({ topic: topic2, partitions });
|
|
2418
|
+
return this.adminOps.deleteRecords(topic2, partitions);
|
|
2170
2419
|
}
|
|
2171
2420
|
/** Return the client ID provided during `KafkaClient` construction. */
|
|
2172
2421
|
getClientId() {
|
|
@@ -2182,39 +2431,19 @@ var KafkaClient = class _KafkaClient {
|
|
|
2182
2431
|
* @returns Read-only `KafkaMetrics` snapshot: `processedCount`, `retryCount`, `dlqCount`, `dedupCount`.
|
|
2183
2432
|
*/
|
|
2184
2433
|
getMetrics(topic2) {
|
|
2185
|
-
|
|
2186
|
-
const m = this._topicMetrics.get(topic2);
|
|
2187
|
-
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
2188
|
-
}
|
|
2189
|
-
const agg = {
|
|
2190
|
-
processedCount: 0,
|
|
2191
|
-
retryCount: 0,
|
|
2192
|
-
dlqCount: 0,
|
|
2193
|
-
dedupCount: 0
|
|
2194
|
-
};
|
|
2195
|
-
for (const m of this._topicMetrics.values()) {
|
|
2196
|
-
agg.processedCount += m.processedCount;
|
|
2197
|
-
agg.retryCount += m.retryCount;
|
|
2198
|
-
agg.dlqCount += m.dlqCount;
|
|
2199
|
-
agg.dedupCount += m.dedupCount;
|
|
2200
|
-
}
|
|
2201
|
-
return agg;
|
|
2434
|
+
return this.metrics.getMetrics(topic2);
|
|
2202
2435
|
}
|
|
2203
2436
|
/**
|
|
2204
2437
|
* Reset internal event counters to zero.
|
|
2205
2438
|
*
|
|
2206
2439
|
* @param topic Topic name to reset. When omitted, all topics are reset.
|
|
2207
|
-
*/
|
|
2208
|
-
resetMetrics(topic2) {
|
|
2209
|
-
|
|
2210
|
-
this._topicMetrics.delete(topic2);
|
|
2211
|
-
return;
|
|
2212
|
-
}
|
|
2213
|
-
this._topicMetrics.clear();
|
|
2440
|
+
*/
|
|
2441
|
+
resetMetrics(topic2) {
|
|
2442
|
+
this.metrics.resetMetrics(topic2);
|
|
2214
2443
|
}
|
|
2215
2444
|
/** Gracefully disconnect producer, all consumers, and admin. */
|
|
2216
2445
|
async disconnect(drainTimeoutMs = 3e4) {
|
|
2217
|
-
await this.waitForDrain(drainTimeoutMs);
|
|
2446
|
+
await this.inFlight.waitForDrain(drainTimeoutMs);
|
|
2218
2447
|
const tasks = [this.producer.disconnect()];
|
|
2219
2448
|
if (this.txProducer) {
|
|
2220
2449
|
tasks.push(this.txProducer.disconnect());
|
|
@@ -2222,28 +2451,17 @@ var KafkaClient = class _KafkaClient {
|
|
|
2222
2451
|
this.txProducer = void 0;
|
|
2223
2452
|
this.txProducerInitPromise = void 0;
|
|
2224
2453
|
}
|
|
2225
|
-
for (const txId of this.retryTxProducers.keys())
|
|
2226
|
-
|
|
2227
|
-
}
|
|
2228
|
-
for (const p of this.retryTxProducers.values()) {
|
|
2229
|
-
tasks.push(p.disconnect());
|
|
2230
|
-
}
|
|
2454
|
+
for (const txId of this.retryTxProducers.keys()) _activeTransactionalIds.delete(txId);
|
|
2455
|
+
for (const p of this.retryTxProducers.values()) tasks.push(p.disconnect());
|
|
2231
2456
|
this.retryTxProducers.clear();
|
|
2232
|
-
for (const consumer of this.consumers.values())
|
|
2233
|
-
|
|
2234
|
-
}
|
|
2235
|
-
if (this.isAdminConnected) {
|
|
2236
|
-
tasks.push(this.admin.disconnect());
|
|
2237
|
-
this.isAdminConnected = false;
|
|
2238
|
-
}
|
|
2457
|
+
for (const consumer of this.consumers.values()) tasks.push(consumer.disconnect());
|
|
2458
|
+
tasks.push(this.adminOps.disconnect());
|
|
2239
2459
|
await Promise.allSettled(tasks);
|
|
2240
2460
|
this.consumers.clear();
|
|
2241
2461
|
this.runningConsumers.clear();
|
|
2242
2462
|
this.consumerCreationOptions.clear();
|
|
2243
2463
|
this.companionGroupIds.clear();
|
|
2244
|
-
|
|
2245
|
-
this.circuitStates.clear();
|
|
2246
|
-
this.circuitConfigs.clear();
|
|
2464
|
+
this.circuitBreaker.clear();
|
|
2247
2465
|
this.logger.log("All connections closed");
|
|
2248
2466
|
}
|
|
2249
2467
|
// ── Graceful shutdown ────────────────────────────────────────────
|
|
@@ -2262,235 +2480,20 @@ var KafkaClient = class _KafkaClient {
|
|
|
2262
2480
|
*/
|
|
2263
2481
|
enableGracefulShutdown(signals = ["SIGTERM", "SIGINT"], drainTimeoutMs = 3e4) {
|
|
2264
2482
|
const handler = () => {
|
|
2265
|
-
this.logger.log(
|
|
2266
|
-
"Shutdown signal received \u2014 draining in-flight handlers..."
|
|
2267
|
-
);
|
|
2483
|
+
this.logger.log("Shutdown signal received \u2014 draining in-flight handlers...");
|
|
2268
2484
|
this.disconnect(drainTimeoutMs).catch(
|
|
2269
|
-
(err) => this.logger.error(
|
|
2270
|
-
"Error during graceful shutdown:",
|
|
2271
|
-
toError(err).message
|
|
2272
|
-
)
|
|
2485
|
+
(err) => this.logger.error("Error during graceful shutdown:", toError(err).message)
|
|
2273
2486
|
);
|
|
2274
2487
|
};
|
|
2275
|
-
for (const signal of signals)
|
|
2276
|
-
process.once(signal, handler);
|
|
2277
|
-
}
|
|
2278
|
-
}
|
|
2279
|
-
/**
|
|
2280
|
-
* Increment the in-flight handler count and return a promise that calls the given handler.
|
|
2281
|
-
* When the promise resolves or rejects, decrement the in flight handler count.
|
|
2282
|
-
* If the in flight handler count reaches 0, call all previously registered drain resolvers.
|
|
2283
|
-
* @param fn The handler to call when the promise is resolved or rejected.
|
|
2284
|
-
* @returns A promise that resolves or rejects with the result of calling the handler.
|
|
2285
|
-
*/
|
|
2286
|
-
trackInFlight(fn) {
|
|
2287
|
-
this.inFlightTotal++;
|
|
2288
|
-
return fn().finally(() => {
|
|
2289
|
-
this.inFlightTotal--;
|
|
2290
|
-
if (this.inFlightTotal === 0) {
|
|
2291
|
-
this.drainResolvers.splice(0).forEach((r) => r());
|
|
2292
|
-
}
|
|
2293
|
-
});
|
|
2294
|
-
}
|
|
2295
|
-
/**
|
|
2296
|
-
* Waits for all in-flight handlers to complete or for a given timeout, whichever comes first.
|
|
2297
|
-
* @param timeoutMs Maximum time to wait in milliseconds.
|
|
2298
|
-
* @returns A promise that resolves when all handlers have completed or the timeout is reached.
|
|
2299
|
-
* @private
|
|
2300
|
-
*/
|
|
2301
|
-
waitForDrain(timeoutMs) {
|
|
2302
|
-
if (this.inFlightTotal === 0) return Promise.resolve();
|
|
2303
|
-
return new Promise((resolve) => {
|
|
2304
|
-
let handle;
|
|
2305
|
-
const onDrain = () => {
|
|
2306
|
-
clearTimeout(handle);
|
|
2307
|
-
resolve();
|
|
2308
|
-
};
|
|
2309
|
-
this.drainResolvers.push(onDrain);
|
|
2310
|
-
handle = setTimeout(() => {
|
|
2311
|
-
const idx = this.drainResolvers.indexOf(onDrain);
|
|
2312
|
-
if (idx !== -1) this.drainResolvers.splice(idx, 1);
|
|
2313
|
-
this.logger.warn(
|
|
2314
|
-
`Drain timed out after ${timeoutMs}ms \u2014 ${this.inFlightTotal} handler(s) still in flight`
|
|
2315
|
-
);
|
|
2316
|
-
resolve();
|
|
2317
|
-
}, timeoutMs);
|
|
2318
|
-
});
|
|
2488
|
+
for (const signal of signals) process.once(signal, handler);
|
|
2319
2489
|
}
|
|
2320
2490
|
// ── Private helpers ──────────────────────────────────────────────
|
|
2321
|
-
/**
|
|
2322
|
-
* Prepare a send payload by registering the topic's schema and then building the payload.
|
|
2323
|
-
* @param topicOrDesc - topic name or topic descriptor
|
|
2324
|
-
* @param messages - batch of messages to send
|
|
2325
|
-
* @returns - prepared payload
|
|
2326
|
-
*/
|
|
2327
2491
|
async preparePayload(topicOrDesc, messages, compression) {
|
|
2328
2492
|
registerSchema(topicOrDesc, this.schemaRegistry, this.logger);
|
|
2329
|
-
const payload = await buildSendPayload(
|
|
2330
|
-
topicOrDesc,
|
|
2331
|
-
messages,
|
|
2332
|
-
this.producerOpsDeps,
|
|
2333
|
-
compression
|
|
2334
|
-
);
|
|
2493
|
+
const payload = await buildSendPayload(topicOrDesc, messages, this._producerOpsDeps, compression);
|
|
2335
2494
|
await this.ensureTopic(payload.topic);
|
|
2336
2495
|
return payload;
|
|
2337
2496
|
}
|
|
2338
|
-
// afterSend is called once per message — symmetric with beforeSend in buildSendPayload.
|
|
2339
|
-
notifyAfterSend(topic2, count) {
|
|
2340
|
-
for (let i = 0; i < count; i++) {
|
|
2341
|
-
for (const inst of this.instrumentation) {
|
|
2342
|
-
inst.afterSend?.(topic2);
|
|
2343
|
-
}
|
|
2344
|
-
}
|
|
2345
|
-
}
|
|
2346
|
-
/**
|
|
2347
|
-
* Returns the KafkaMetrics for a given topic.
|
|
2348
|
-
* If the topic hasn't seen any events, initializes a zero-valued snapshot.
|
|
2349
|
-
* @param topic - name of the topic to get the metrics for
|
|
2350
|
-
* @returns - KafkaMetrics for the given topic
|
|
2351
|
-
*/
|
|
2352
|
-
metricsFor(topic2) {
|
|
2353
|
-
let m = this._topicMetrics.get(topic2);
|
|
2354
|
-
if (!m) {
|
|
2355
|
-
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
2356
|
-
this._topicMetrics.set(topic2, m);
|
|
2357
|
-
}
|
|
2358
|
-
return m;
|
|
2359
|
-
}
|
|
2360
|
-
/**
|
|
2361
|
-
* Notifies instrumentation hooks of a retry event.
|
|
2362
|
-
* @param envelope The original message envelope that triggered the retry.
|
|
2363
|
-
* @param attempt The current retry attempt (1-indexed).
|
|
2364
|
-
* @param maxRetries The maximum number of retries configured for this topic.
|
|
2365
|
-
*/
|
|
2366
|
-
notifyRetry(envelope, attempt, maxRetries) {
|
|
2367
|
-
this.metricsFor(envelope.topic).retryCount++;
|
|
2368
|
-
for (const inst of this.instrumentation) {
|
|
2369
|
-
inst.onRetry?.(envelope, attempt, maxRetries);
|
|
2370
|
-
}
|
|
2371
|
-
}
|
|
2372
|
-
/**
|
|
2373
|
-
* Called whenever a message is routed to the dead letter queue.
|
|
2374
|
-
* @param envelope The original message envelope.
|
|
2375
|
-
* @param reason The reason for routing to the dead letter queue.
|
|
2376
|
-
* @param gid The group ID of the consumer that triggered the circuit breaker, if any.
|
|
2377
|
-
*/
|
|
2378
|
-
notifyDlq(envelope, reason, gid) {
|
|
2379
|
-
this.metricsFor(envelope.topic).dlqCount++;
|
|
2380
|
-
for (const inst of this.instrumentation) {
|
|
2381
|
-
inst.onDlq?.(envelope, reason);
|
|
2382
|
-
}
|
|
2383
|
-
if (!gid) return;
|
|
2384
|
-
const cfg = this.circuitConfigs.get(gid);
|
|
2385
|
-
if (!cfg) return;
|
|
2386
|
-
const threshold = cfg.threshold ?? 5;
|
|
2387
|
-
const recoveryMs = cfg.recoveryMs ?? 3e4;
|
|
2388
|
-
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
2389
|
-
let state = this.circuitStates.get(stateKey);
|
|
2390
|
-
if (!state) {
|
|
2391
|
-
state = { status: "closed", window: [], successes: 0 };
|
|
2392
|
-
this.circuitStates.set(stateKey, state);
|
|
2393
|
-
}
|
|
2394
|
-
if (state.status === "open") return;
|
|
2395
|
-
const openCircuit = () => {
|
|
2396
|
-
state.status = "open";
|
|
2397
|
-
state.window = [];
|
|
2398
|
-
state.successes = 0;
|
|
2399
|
-
clearTimeout(state.timer);
|
|
2400
|
-
for (const inst of this.instrumentation)
|
|
2401
|
-
inst.onCircuitOpen?.(envelope.topic, envelope.partition);
|
|
2402
|
-
this.pauseConsumer(gid, [
|
|
2403
|
-
{ topic: envelope.topic, partitions: [envelope.partition] }
|
|
2404
|
-
]);
|
|
2405
|
-
state.timer = setTimeout(() => {
|
|
2406
|
-
state.status = "half-open";
|
|
2407
|
-
state.successes = 0;
|
|
2408
|
-
this.logger.log(
|
|
2409
|
-
`[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2410
|
-
);
|
|
2411
|
-
for (const inst of this.instrumentation)
|
|
2412
|
-
inst.onCircuitHalfOpen?.(envelope.topic, envelope.partition);
|
|
2413
|
-
this.resumeConsumer(gid, [
|
|
2414
|
-
{ topic: envelope.topic, partitions: [envelope.partition] }
|
|
2415
|
-
]);
|
|
2416
|
-
}, recoveryMs);
|
|
2417
|
-
};
|
|
2418
|
-
if (state.status === "half-open") {
|
|
2419
|
-
clearTimeout(state.timer);
|
|
2420
|
-
this.logger.warn(
|
|
2421
|
-
`[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2422
|
-
);
|
|
2423
|
-
openCircuit();
|
|
2424
|
-
return;
|
|
2425
|
-
}
|
|
2426
|
-
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
2427
|
-
state.window = [...state.window, false];
|
|
2428
|
-
if (state.window.length > windowSize) {
|
|
2429
|
-
state.window = state.window.slice(state.window.length - windowSize);
|
|
2430
|
-
}
|
|
2431
|
-
const failures = state.window.filter((v) => !v).length;
|
|
2432
|
-
if (failures >= threshold) {
|
|
2433
|
-
this.logger.warn(
|
|
2434
|
-
`[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
|
|
2435
|
-
);
|
|
2436
|
-
openCircuit();
|
|
2437
|
-
}
|
|
2438
|
-
}
|
|
2439
|
-
/**
|
|
2440
|
-
* Notify all instrumentation hooks about a duplicate message detection.
|
|
2441
|
-
* Invoked by the consumer after a message has been successfully processed
|
|
2442
|
-
* and the Lamport clock detected a duplicate.
|
|
2443
|
-
* @param envelope The processed message envelope.
|
|
2444
|
-
* @param strategy The duplicate detection strategy used.
|
|
2445
|
-
*/
|
|
2446
|
-
notifyDuplicate(envelope, strategy) {
|
|
2447
|
-
this.metricsFor(envelope.topic).dedupCount++;
|
|
2448
|
-
for (const inst of this.instrumentation) {
|
|
2449
|
-
inst.onDuplicate?.(envelope, strategy);
|
|
2450
|
-
}
|
|
2451
|
-
}
|
|
2452
|
-
/**
|
|
2453
|
-
* Notify all instrumentation hooks about a successfully processed message.
|
|
2454
|
-
* Invoked by the consumer after a message has been successfully processed
|
|
2455
|
-
* by the handler.
|
|
2456
|
-
* @param envelope The processed message envelope.
|
|
2457
|
-
* @param gid The optional consumer group ID.
|
|
2458
|
-
*/
|
|
2459
|
-
notifyMessage(envelope, gid) {
|
|
2460
|
-
this.metricsFor(envelope.topic).processedCount++;
|
|
2461
|
-
for (const inst of this.instrumentation) {
|
|
2462
|
-
inst.onMessage?.(envelope);
|
|
2463
|
-
}
|
|
2464
|
-
if (!gid) return;
|
|
2465
|
-
const cfg = this.circuitConfigs.get(gid);
|
|
2466
|
-
if (!cfg) return;
|
|
2467
|
-
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
2468
|
-
const state = this.circuitStates.get(stateKey);
|
|
2469
|
-
if (!state) return;
|
|
2470
|
-
const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
|
|
2471
|
-
if (state.status === "half-open") {
|
|
2472
|
-
state.successes++;
|
|
2473
|
-
if (state.successes >= halfOpenSuccesses) {
|
|
2474
|
-
clearTimeout(state.timer);
|
|
2475
|
-
state.timer = void 0;
|
|
2476
|
-
state.status = "closed";
|
|
2477
|
-
state.window = [];
|
|
2478
|
-
state.successes = 0;
|
|
2479
|
-
this.logger.log(
|
|
2480
|
-
`[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2481
|
-
);
|
|
2482
|
-
for (const inst of this.instrumentation)
|
|
2483
|
-
inst.onCircuitClose?.(envelope.topic, envelope.partition);
|
|
2484
|
-
}
|
|
2485
|
-
} else if (state.status === "closed") {
|
|
2486
|
-
const threshold = cfg.threshold ?? 5;
|
|
2487
|
-
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
2488
|
-
state.window = [...state.window, true];
|
|
2489
|
-
if (state.window.length > windowSize) {
|
|
2490
|
-
state.window = state.window.slice(state.window.length - windowSize);
|
|
2491
|
-
}
|
|
2492
|
-
}
|
|
2493
|
-
}
|
|
2494
2497
|
/**
|
|
2495
2498
|
* Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
|
|
2496
2499
|
* The handler itself is not cancelled — the warning is diagnostic only.
|
|
@@ -2501,79 +2504,10 @@ var KafkaClient = class _KafkaClient {
|
|
|
2501
2504
|
if (timer !== void 0) clearTimeout(timer);
|
|
2502
2505
|
});
|
|
2503
2506
|
timer = setTimeout(() => {
|
|
2504
|
-
this.logger.warn(
|
|
2505
|
-
`Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
|
|
2506
|
-
);
|
|
2507
|
+
this.logger.warn(`Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`);
|
|
2507
2508
|
}, timeoutMs);
|
|
2508
2509
|
return promise;
|
|
2509
2510
|
}
|
|
2510
|
-
/**
|
|
2511
|
-
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
2512
|
-
* `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
|
|
2513
|
-
* rather than silently discovering missing topics on the first handler failure.
|
|
2514
|
-
*/
|
|
2515
|
-
async validateRetryTopicsExist(topicNames, maxRetries) {
|
|
2516
|
-
await this.ensureAdminConnected();
|
|
2517
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2518
|
-
const missing = [];
|
|
2519
|
-
for (const t of topicNames) {
|
|
2520
|
-
for (let level = 1; level <= maxRetries; level++) {
|
|
2521
|
-
const retryTopic = `${t}.retry.${level}`;
|
|
2522
|
-
if (!existing.has(retryTopic)) missing.push(retryTopic);
|
|
2523
|
-
}
|
|
2524
|
-
}
|
|
2525
|
-
if (missing.length > 0) {
|
|
2526
|
-
throw new Error(
|
|
2527
|
-
`retryTopics: true but the following retry topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2528
|
-
);
|
|
2529
|
-
}
|
|
2530
|
-
}
|
|
2531
|
-
/**
|
|
2532
|
-
* When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
|
|
2533
|
-
* consumed topic. Throws a clear error at startup rather than silently discovering
|
|
2534
|
-
* missing DLQ topics on the first handler failure.
|
|
2535
|
-
*/
|
|
2536
|
-
async validateDlqTopicsExist(topicNames) {
|
|
2537
|
-
await this.ensureAdminConnected();
|
|
2538
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2539
|
-
const missing = topicNames.filter((t) => !existing.has(`${t}.dlq`)).map((t) => `${t}.dlq`);
|
|
2540
|
-
if (missing.length > 0) {
|
|
2541
|
-
throw new Error(
|
|
2542
|
-
`dlq: true but the following DLQ topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2543
|
-
);
|
|
2544
|
-
}
|
|
2545
|
-
}
|
|
2546
|
-
/**
|
|
2547
|
-
* When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
|
|
2548
|
-
* that every `<topic>.duplicates` destination topic already exists. Throws a
|
|
2549
|
-
* clear error at startup rather than silently dropping duplicates on first hit.
|
|
2550
|
-
*/
|
|
2551
|
-
async validateDuplicatesTopicsExist(topicNames, customDestination) {
|
|
2552
|
-
await this.ensureAdminConnected();
|
|
2553
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2554
|
-
const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
|
|
2555
|
-
const missing = toCheck.filter((t) => !existing.has(t));
|
|
2556
|
-
if (missing.length > 0) {
|
|
2557
|
-
throw new Error(
|
|
2558
|
-
`deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2559
|
-
);
|
|
2560
|
-
}
|
|
2561
|
-
}
|
|
2562
|
-
/**
|
|
2563
|
-
* Connect the admin client if not already connected.
|
|
2564
|
-
* The flag is only set to `true` after a successful connect — if `admin.connect()`
|
|
2565
|
-
* throws the flag remains `false` so the next call will retry the connection.
|
|
2566
|
-
*/
|
|
2567
|
-
async ensureAdminConnected() {
|
|
2568
|
-
if (this.isAdminConnected) return;
|
|
2569
|
-
try {
|
|
2570
|
-
await this.admin.connect();
|
|
2571
|
-
this.isAdminConnected = true;
|
|
2572
|
-
} catch (err) {
|
|
2573
|
-
this.isAdminConnected = false;
|
|
2574
|
-
throw err;
|
|
2575
|
-
}
|
|
2576
|
-
}
|
|
2577
2511
|
/**
|
|
2578
2512
|
* Create and connect a transactional producer for EOS retry routing.
|
|
2579
2513
|
* Each retry level consumer gets its own producer with a unique `transactionalId`
|
|
@@ -2586,12 +2520,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
2586
2520
|
);
|
|
2587
2521
|
}
|
|
2588
2522
|
const p = this.kafka.producer({
|
|
2589
|
-
kafkaJS: {
|
|
2590
|
-
acks: -1,
|
|
2591
|
-
idempotent: true,
|
|
2592
|
-
transactionalId,
|
|
2593
|
-
maxInFlightRequests: 1
|
|
2594
|
-
}
|
|
2523
|
+
kafkaJS: { acks: -1, idempotent: true, transactionalId, maxInFlightRequests: 1 }
|
|
2595
2524
|
});
|
|
2596
2525
|
await p.connect();
|
|
2597
2526
|
_activeTransactionalIds.add(transactionalId);
|
|
@@ -2600,20 +2529,16 @@ var KafkaClient = class _KafkaClient {
|
|
|
2600
2529
|
}
|
|
2601
2530
|
/**
|
|
2602
2531
|
* Ensure that a topic exists by creating it if it doesn't already exist.
|
|
2603
|
-
* If `autoCreateTopics` is disabled,
|
|
2604
|
-
*
|
|
2605
|
-
* If multiple concurrent calls are made to `ensureTopic` for the same topic,
|
|
2606
|
-
* they are deduplicated to prevent multiple calls to `admin.createTopics()`.
|
|
2607
|
-
* @param topic - The topic to ensure exists.
|
|
2608
|
-
* @returns A promise that resolves when the topic has been created or already exists.
|
|
2532
|
+
* If `autoCreateTopics` is disabled, returns immediately.
|
|
2533
|
+
* Concurrent calls for the same topic are deduplicated.
|
|
2609
2534
|
*/
|
|
2610
2535
|
async ensureTopic(topic2) {
|
|
2611
2536
|
if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
|
|
2612
2537
|
let p = this.ensureTopicPromises.get(topic2);
|
|
2613
2538
|
if (!p) {
|
|
2614
2539
|
p = (async () => {
|
|
2615
|
-
await this.
|
|
2616
|
-
await this.admin.createTopics({
|
|
2540
|
+
await this.adminOps.ensureConnected();
|
|
2541
|
+
await this.adminOps.admin.createTopics({
|
|
2617
2542
|
topics: [{ topic: topic2, numPartitions: this.numPartitions }]
|
|
2618
2543
|
});
|
|
2619
2544
|
this.ensuredTopics.add(topic2);
|
|
@@ -2653,99 +2578,78 @@ var KafkaClient = class _KafkaClient {
|
|
|
2653
2578
|
gid,
|
|
2654
2579
|
fromBeginning,
|
|
2655
2580
|
options.autoCommit ?? true,
|
|
2656
|
-
this.
|
|
2581
|
+
this._consumerOpsDeps,
|
|
2657
2582
|
options.partitionAssigner
|
|
2658
2583
|
);
|
|
2659
|
-
const schemaMap = buildSchemaMap(
|
|
2660
|
-
stringTopics,
|
|
2661
|
-
this.schemaRegistry,
|
|
2662
|
-
optionSchemas,
|
|
2663
|
-
this.logger
|
|
2664
|
-
);
|
|
2584
|
+
const schemaMap = buildSchemaMap(stringTopics, this.schemaRegistry, optionSchemas, this.logger);
|
|
2665
2585
|
const topicNames = stringTopics.map((t) => resolveTopicName(t));
|
|
2666
|
-
const subscribeTopics = [
|
|
2667
|
-
|
|
2668
|
-
...regexTopics
|
|
2669
|
-
];
|
|
2670
|
-
for (const t of topicNames) {
|
|
2671
|
-
await this.ensureTopic(t);
|
|
2672
|
-
}
|
|
2586
|
+
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2587
|
+
for (const t of topicNames) await this.ensureTopic(t);
|
|
2673
2588
|
if (dlq) {
|
|
2674
|
-
for (const t of topicNames) {
|
|
2675
|
-
await this.ensureTopic(`${t}.dlq`);
|
|
2676
|
-
}
|
|
2589
|
+
for (const t of topicNames) await this.ensureTopic(`${t}.dlq`);
|
|
2677
2590
|
if (!this.autoCreateTopicsEnabled && topicNames.length > 0) {
|
|
2678
|
-
await this.validateDlqTopicsExist(topicNames);
|
|
2591
|
+
await this.adminOps.validateDlqTopicsExist(topicNames);
|
|
2679
2592
|
}
|
|
2680
2593
|
}
|
|
2681
2594
|
if (options.deduplication?.strategy === "topic") {
|
|
2682
2595
|
const dest = options.deduplication.duplicatesTopic;
|
|
2683
2596
|
if (this.autoCreateTopicsEnabled) {
|
|
2684
|
-
for (const t of topicNames) {
|
|
2685
|
-
await this.ensureTopic(dest ?? `${t}.duplicates`);
|
|
2686
|
-
}
|
|
2597
|
+
for (const t of topicNames) await this.ensureTopic(dest ?? `${t}.duplicates`);
|
|
2687
2598
|
} else if (topicNames.length > 0) {
|
|
2688
|
-
await this.validateDuplicatesTopicsExist(topicNames, dest);
|
|
2599
|
+
await this.adminOps.validateDuplicatesTopicsExist(topicNames, dest);
|
|
2689
2600
|
}
|
|
2690
2601
|
}
|
|
2691
2602
|
await consumer.connect();
|
|
2692
|
-
await subscribeWithRetry(
|
|
2693
|
-
consumer,
|
|
2694
|
-
subscribeTopics,
|
|
2695
|
-
this.logger,
|
|
2696
|
-
options.subscribeRetry
|
|
2697
|
-
);
|
|
2603
|
+
await subscribeWithRetry(consumer, subscribeTopics, this.logger, options.subscribeRetry);
|
|
2698
2604
|
const displayTopics = subscribeTopics.map((t) => t instanceof RegExp ? t.toString() : t).join(", ");
|
|
2699
|
-
this.logger.log(
|
|
2700
|
-
`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
|
|
2701
|
-
);
|
|
2605
|
+
this.logger.log(`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`);
|
|
2702
2606
|
return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex };
|
|
2703
2607
|
}
|
|
2704
2608
|
/** Create or retrieve the deduplication context for a consumer group. */
|
|
2705
2609
|
resolveDeduplicationContext(groupId, options) {
|
|
2706
2610
|
if (!options) return void 0;
|
|
2707
|
-
if (!this.dedupStates.has(groupId))
|
|
2708
|
-
this.dedupStates.set(groupId, /* @__PURE__ */ new Map());
|
|
2709
|
-
}
|
|
2611
|
+
if (!this.dedupStates.has(groupId)) this.dedupStates.set(groupId, /* @__PURE__ */ new Map());
|
|
2710
2612
|
return { options, state: this.dedupStates.get(groupId) };
|
|
2711
2613
|
}
|
|
2712
|
-
// ──
|
|
2713
|
-
/**
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
strictSchemasEnabled: this.strictSchemasEnabled,
|
|
2726
|
-
instrumentation: this.instrumentation,
|
|
2727
|
-
logger: this.logger,
|
|
2728
|
-
nextLamportClock: () => ++this._lamportClock
|
|
2729
|
-
};
|
|
2614
|
+
// ── Shared consumer setup helpers ────────────────────────────────
|
|
2615
|
+
/** Guard checks shared by startConsumer and startBatchConsumer. */
|
|
2616
|
+
validateTopicConsumerOpts(topics, options) {
|
|
2617
|
+
if (options.retryTopics && !options.retry) {
|
|
2618
|
+
throw new Error(
|
|
2619
|
+
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
2620
|
+
);
|
|
2621
|
+
}
|
|
2622
|
+
if (options.retryTopics && topics.some((t) => t instanceof RegExp)) {
|
|
2623
|
+
throw new Error(
|
|
2624
|
+
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
2625
|
+
);
|
|
2626
|
+
}
|
|
2730
2627
|
}
|
|
2731
|
-
/**
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2628
|
+
/** Create EOS transactional producer context for atomic main → retry.1 routing. */
|
|
2629
|
+
async makeEosMainContext(gid, consumer, options) {
|
|
2630
|
+
if (!options.retryTopics || !options.retry) return void 0;
|
|
2631
|
+
const txProducer = await this.createRetryTxProducer(`${gid}-main-tx`);
|
|
2632
|
+
return { txProducer, consumer };
|
|
2633
|
+
}
|
|
2634
|
+
/** Start companion retry-level consumers and register them under the main groupId. */
|
|
2635
|
+
async launchRetryChain(gid, topicNames, handleMessage, retry, dlq, interceptors, schemaMap, assignmentTimeoutMs) {
|
|
2636
|
+
if (!this.autoCreateTopicsEnabled) {
|
|
2637
|
+
await this.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
2638
|
+
}
|
|
2639
|
+
const companions = await startRetryTopicConsumers(
|
|
2640
|
+
topicNames,
|
|
2641
|
+
gid,
|
|
2642
|
+
handleMessage,
|
|
2643
|
+
retry,
|
|
2644
|
+
dlq,
|
|
2645
|
+
interceptors,
|
|
2646
|
+
schemaMap,
|
|
2647
|
+
this._retryTopicDeps,
|
|
2648
|
+
assignmentTimeoutMs
|
|
2649
|
+
);
|
|
2650
|
+
this.companionGroupIds.set(gid, companions);
|
|
2748
2651
|
}
|
|
2652
|
+
// ── Deps object builders ─────────────────────────────────────────
|
|
2749
2653
|
/** Build MessageHandlerDeps with circuit breaker callbacks bound to the given groupId. */
|
|
2750
2654
|
messageDepsFor(gid) {
|
|
2751
2655
|
return {
|
|
@@ -2754,38 +2658,24 @@ var KafkaClient = class _KafkaClient {
|
|
|
2754
2658
|
instrumentation: this.instrumentation,
|
|
2755
2659
|
onMessageLost: this.onMessageLost,
|
|
2756
2660
|
onTtlExpired: this.onTtlExpired,
|
|
2757
|
-
onRetry: this.notifyRetry.bind(this),
|
|
2758
|
-
onDlq: (envelope, reason) => this.notifyDlq(envelope, reason, gid),
|
|
2759
|
-
onDuplicate: this.notifyDuplicate.bind(this),
|
|
2760
|
-
onMessage: (envelope) => this.notifyMessage(envelope, gid)
|
|
2661
|
+
onRetry: this.metrics.notifyRetry.bind(this.metrics),
|
|
2662
|
+
onDlq: (envelope, reason) => this.metrics.notifyDlq(envelope, reason, gid),
|
|
2663
|
+
onDuplicate: this.metrics.notifyDuplicate.bind(this.metrics),
|
|
2664
|
+
onMessage: (envelope) => this.metrics.notifyMessage(envelope, gid)
|
|
2761
2665
|
};
|
|
2762
2666
|
}
|
|
2763
|
-
/**
|
|
2764
|
-
|
|
2765
|
-
*
|
|
2766
|
-
* `logger`: The logger instance passed to the retry topic consumers.
|
|
2767
|
-
* `producer`: The producer instance passed to the retry topic consumers.
|
|
2768
|
-
* `instrumentation`: The instrumentation instance passed to the retry topic consumers.
|
|
2769
|
-
* `onMessageLost`: The callback function passed to the retry topic consumers for tracking lost messages.
|
|
2770
|
-
* `onRetry`: The callback function passed to the retry topic consumers for tracking retry attempts.
|
|
2771
|
-
* `onDlq`: The callback function passed to the retry topic consumers for tracking dead-letter queue routing.
|
|
2772
|
-
* `onMessage`: The callback function passed to the retry topic consumers for tracking message delivery.
|
|
2773
|
-
* `ensureTopic`: A function that ensures a topic exists before subscribing to it.
|
|
2774
|
-
* `getOrCreateConsumer`: A function that creates or retrieves a consumer instance.
|
|
2775
|
-
* `runningConsumers`: A map of consumer group IDs to their corresponding consumer instances.
|
|
2776
|
-
* `createRetryTxProducer`: A function that creates a retry transactional producer instance.
|
|
2777
|
-
*/
|
|
2778
|
-
get retryTopicDeps() {
|
|
2667
|
+
/** Build the deps object passed to retry topic consumers. */
|
|
2668
|
+
buildRetryTopicDeps() {
|
|
2779
2669
|
return {
|
|
2780
2670
|
logger: this.logger,
|
|
2781
2671
|
producer: this.producer,
|
|
2782
2672
|
instrumentation: this.instrumentation,
|
|
2783
2673
|
onMessageLost: this.onMessageLost,
|
|
2784
|
-
onRetry: this.notifyRetry.bind(this),
|
|
2785
|
-
onDlq: this.notifyDlq.bind(this),
|
|
2786
|
-
onMessage: this.notifyMessage.bind(this),
|
|
2674
|
+
onRetry: this.metrics.notifyRetry.bind(this.metrics),
|
|
2675
|
+
onDlq: this.metrics.notifyDlq.bind(this.metrics),
|
|
2676
|
+
onMessage: this.metrics.notifyMessage.bind(this.metrics),
|
|
2787
2677
|
ensureTopic: (t) => this.ensureTopic(t),
|
|
2788
|
-
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.
|
|
2678
|
+
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this._consumerOpsDeps),
|
|
2789
2679
|
runningConsumers: this.runningConsumers,
|
|
2790
2680
|
createRetryTxProducer: (txId) => this.createRetryTxProducer(txId)
|
|
2791
2681
|
};
|