@drarzter/kafka-client 0.7.2 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/{chunk-MADAJD2F.mjs → chunk-XP7LLRGQ.mjs} +751 -861
- package/dist/chunk-XP7LLRGQ.mjs.map +1 -0
- package/dist/core.d.mts +19 -137
- package/dist/core.d.ts +19 -137
- package/dist/core.js +750 -860
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +1 -1
- package/dist/index.js +751 -861
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
- package/dist/chunk-MADAJD2F.mjs.map +0 -1
|
@@ -95,7 +95,7 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
|
95
95
|
}
|
|
96
96
|
};
|
|
97
97
|
|
|
98
|
-
// src/client/kafka.client/producer
|
|
98
|
+
// src/client/kafka.client/producer/ops.ts
|
|
99
99
|
function resolveTopicName(topicOrDescriptor) {
|
|
100
100
|
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
101
101
|
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
@@ -173,7 +173,7 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
173
173
|
return { topic: topic2, messages: builtMessages, ...compression && { compression } };
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
// src/client/kafka.client/consumer
|
|
176
|
+
// src/client/kafka.client/consumer/ops.ts
|
|
177
177
|
import { KafkaJS } from "@confluentinc/kafka-javascript";
|
|
178
178
|
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner) {
|
|
179
179
|
const { consumers, consumerCreationOptions, kafka, onRebalance, logger } = deps;
|
|
@@ -236,7 +236,7 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
|
|
|
236
236
|
return schemaMap;
|
|
237
237
|
}
|
|
238
238
|
|
|
239
|
-
// src/client/consumer/pipeline.ts
|
|
239
|
+
// src/client/kafka.client/consumer/pipeline.ts
|
|
240
240
|
function toError(error) {
|
|
241
241
|
return error instanceof Error ? error : new Error(String(error));
|
|
242
242
|
}
|
|
@@ -586,7 +586,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
586
586
|
}
|
|
587
587
|
}
|
|
588
588
|
|
|
589
|
-
// src/client/kafka.client/
|
|
589
|
+
// src/client/kafka.client/consumer/handler.ts
|
|
590
590
|
async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
591
591
|
const clockRaw = envelope.headers[HEADER_LAMPORT_CLOCK];
|
|
592
592
|
if (clockRaw === void 0) return false;
|
|
@@ -907,7 +907,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
907
907
|
);
|
|
908
908
|
}
|
|
909
909
|
|
|
910
|
-
// src/client/consumer/subscribe-retry.ts
|
|
910
|
+
// src/client/kafka.client/consumer/subscribe-retry.ts
|
|
911
911
|
async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
912
912
|
const maxAttempts = retryOpts?.retries ?? 5;
|
|
913
913
|
const backoffMs = retryOpts?.backoffMs ?? 5e3;
|
|
@@ -928,7 +928,7 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
|
928
928
|
}
|
|
929
929
|
}
|
|
930
930
|
|
|
931
|
-
// src/client/kafka.client/retry-topic.ts
|
|
931
|
+
// src/client/kafka.client/consumer/retry-topic.ts
|
|
932
932
|
async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs = 1e4) {
|
|
933
933
|
const topicSet = new Set(topics);
|
|
934
934
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -1182,9 +1182,7 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
1182
1182
|
return levelGroupIds;
|
|
1183
1183
|
}
|
|
1184
1184
|
|
|
1185
|
-
// src/client/kafka.client/
|
|
1186
|
-
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS2;
|
|
1187
|
-
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1185
|
+
// src/client/kafka.client/consumer/queue.ts
|
|
1188
1186
|
var AsyncQueue = class {
|
|
1189
1187
|
constructor(highWaterMark = Infinity, onFull = () => {
|
|
1190
1188
|
}, onDrained = () => {
|
|
@@ -1233,6 +1231,562 @@ var AsyncQueue = class {
|
|
|
1233
1231
|
return new Promise((resolve, reject) => this.waiting.push({ resolve, reject }));
|
|
1234
1232
|
}
|
|
1235
1233
|
};
|
|
1234
|
+
|
|
1235
|
+
// src/client/kafka.client/infra/circuit-breaker.ts
|
|
1236
|
+
var CircuitBreakerManager = class {
|
|
1237
|
+
constructor(deps) {
|
|
1238
|
+
this.deps = deps;
|
|
1239
|
+
}
|
|
1240
|
+
states = /* @__PURE__ */ new Map();
|
|
1241
|
+
configs = /* @__PURE__ */ new Map();
|
|
1242
|
+
setConfig(gid, options) {
|
|
1243
|
+
this.configs.set(gid, options);
|
|
1244
|
+
}
|
|
1245
|
+
/**
|
|
1246
|
+
* Returns a snapshot of the circuit breaker state for a given topic-partition.
|
|
1247
|
+
* Returns `undefined` when no state exists for the key.
|
|
1248
|
+
*/
|
|
1249
|
+
getState(topic2, partition, gid) {
|
|
1250
|
+
const state = this.states.get(`${gid}:${topic2}:${partition}`);
|
|
1251
|
+
if (!state) return void 0;
|
|
1252
|
+
return {
|
|
1253
|
+
status: state.status,
|
|
1254
|
+
failures: state.window.filter((v) => !v).length,
|
|
1255
|
+
windowSize: state.window.length
|
|
1256
|
+
};
|
|
1257
|
+
}
|
|
1258
|
+
/**
|
|
1259
|
+
* Record a failure for the given envelope and group.
|
|
1260
|
+
* Drives the CLOSED → OPEN and HALF-OPEN → OPEN transitions.
|
|
1261
|
+
*/
|
|
1262
|
+
onFailure(envelope, gid) {
|
|
1263
|
+
const cfg = this.configs.get(gid);
|
|
1264
|
+
if (!cfg) return;
|
|
1265
|
+
const threshold = cfg.threshold ?? 5;
|
|
1266
|
+
const recoveryMs = cfg.recoveryMs ?? 3e4;
|
|
1267
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
1268
|
+
let state = this.states.get(stateKey);
|
|
1269
|
+
if (!state) {
|
|
1270
|
+
state = { status: "closed", window: [], successes: 0 };
|
|
1271
|
+
this.states.set(stateKey, state);
|
|
1272
|
+
}
|
|
1273
|
+
if (state.status === "open") return;
|
|
1274
|
+
const openCircuit = () => {
|
|
1275
|
+
state.status = "open";
|
|
1276
|
+
state.window = [];
|
|
1277
|
+
state.successes = 0;
|
|
1278
|
+
clearTimeout(state.timer);
|
|
1279
|
+
for (const inst of this.deps.instrumentation)
|
|
1280
|
+
inst.onCircuitOpen?.(envelope.topic, envelope.partition);
|
|
1281
|
+
this.deps.pauseConsumer(gid, [{ topic: envelope.topic, partitions: [envelope.partition] }]);
|
|
1282
|
+
state.timer = setTimeout(() => {
|
|
1283
|
+
state.status = "half-open";
|
|
1284
|
+
state.successes = 0;
|
|
1285
|
+
this.deps.logger.log(
|
|
1286
|
+
`[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1287
|
+
);
|
|
1288
|
+
for (const inst of this.deps.instrumentation)
|
|
1289
|
+
inst.onCircuitHalfOpen?.(envelope.topic, envelope.partition);
|
|
1290
|
+
this.deps.resumeConsumer(gid, [{ topic: envelope.topic, partitions: [envelope.partition] }]);
|
|
1291
|
+
}, recoveryMs);
|
|
1292
|
+
};
|
|
1293
|
+
if (state.status === "half-open") {
|
|
1294
|
+
clearTimeout(state.timer);
|
|
1295
|
+
this.deps.logger.warn(
|
|
1296
|
+
`[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1297
|
+
);
|
|
1298
|
+
openCircuit();
|
|
1299
|
+
return;
|
|
1300
|
+
}
|
|
1301
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
1302
|
+
state.window = [...state.window, false];
|
|
1303
|
+
if (state.window.length > windowSize) {
|
|
1304
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
1305
|
+
}
|
|
1306
|
+
const failures = state.window.filter((v) => !v).length;
|
|
1307
|
+
if (failures >= threshold) {
|
|
1308
|
+
this.deps.logger.warn(
|
|
1309
|
+
`[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
|
|
1310
|
+
);
|
|
1311
|
+
openCircuit();
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
/**
|
|
1315
|
+
* Record a success for the given envelope and group.
|
|
1316
|
+
* Drives the HALF-OPEN → CLOSED transition and updates the success window.
|
|
1317
|
+
*/
|
|
1318
|
+
onSuccess(envelope, gid) {
|
|
1319
|
+
const cfg = this.configs.get(gid);
|
|
1320
|
+
if (!cfg) return;
|
|
1321
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
1322
|
+
const state = this.states.get(stateKey);
|
|
1323
|
+
if (!state) return;
|
|
1324
|
+
const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
|
|
1325
|
+
if (state.status === "half-open") {
|
|
1326
|
+
state.successes++;
|
|
1327
|
+
if (state.successes >= halfOpenSuccesses) {
|
|
1328
|
+
clearTimeout(state.timer);
|
|
1329
|
+
state.timer = void 0;
|
|
1330
|
+
state.status = "closed";
|
|
1331
|
+
state.window = [];
|
|
1332
|
+
state.successes = 0;
|
|
1333
|
+
this.deps.logger.log(
|
|
1334
|
+
`[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1335
|
+
);
|
|
1336
|
+
for (const inst of this.deps.instrumentation)
|
|
1337
|
+
inst.onCircuitClose?.(envelope.topic, envelope.partition);
|
|
1338
|
+
}
|
|
1339
|
+
} else if (state.status === "closed") {
|
|
1340
|
+
const threshold = cfg.threshold ?? 5;
|
|
1341
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
1342
|
+
state.window = [...state.window, true];
|
|
1343
|
+
if (state.window.length > windowSize) {
|
|
1344
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
/**
|
|
1349
|
+
* Remove all circuit state and config for the given group.
|
|
1350
|
+
* Called when a consumer is stopped via `stopConsumer(groupId)`.
|
|
1351
|
+
*/
|
|
1352
|
+
removeGroup(gid) {
|
|
1353
|
+
for (const key of [...this.states.keys()]) {
|
|
1354
|
+
if (key.startsWith(`${gid}:`)) {
|
|
1355
|
+
clearTimeout(this.states.get(key).timer);
|
|
1356
|
+
this.states.delete(key);
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
this.configs.delete(gid);
|
|
1360
|
+
}
|
|
1361
|
+
/** Clear all circuit state and config. Called on `disconnect()`. */
|
|
1362
|
+
clear() {
|
|
1363
|
+
for (const state of this.states.values()) clearTimeout(state.timer);
|
|
1364
|
+
this.states.clear();
|
|
1365
|
+
this.configs.clear();
|
|
1366
|
+
}
|
|
1367
|
+
};
|
|
1368
|
+
|
|
1369
|
+
// src/client/kafka.client/admin/ops.ts
|
|
1370
|
+
var AdminOps = class {
|
|
1371
|
+
constructor(deps) {
|
|
1372
|
+
this.deps = deps;
|
|
1373
|
+
}
|
|
1374
|
+
isConnected = false;
|
|
1375
|
+
/** Underlying admin client — used by index.ts for topic validation. */
|
|
1376
|
+
get admin() {
|
|
1377
|
+
return this.deps.admin;
|
|
1378
|
+
}
|
|
1379
|
+
/** Whether the admin client is currently connected. */
|
|
1380
|
+
get connected() {
|
|
1381
|
+
return this.isConnected;
|
|
1382
|
+
}
|
|
1383
|
+
/**
|
|
1384
|
+
* Connect the admin client if not already connected.
|
|
1385
|
+
* The flag is only set to `true` after a successful connect — if `admin.connect()`
|
|
1386
|
+
* throws the flag remains `false` so the next call will retry the connection.
|
|
1387
|
+
*/
|
|
1388
|
+
async ensureConnected() {
|
|
1389
|
+
if (this.isConnected) return;
|
|
1390
|
+
try {
|
|
1391
|
+
await this.deps.admin.connect();
|
|
1392
|
+
this.isConnected = true;
|
|
1393
|
+
} catch (err) {
|
|
1394
|
+
this.isConnected = false;
|
|
1395
|
+
throw err;
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
/** Disconnect admin if connected. Resets the connected flag. */
|
|
1399
|
+
async disconnect() {
|
|
1400
|
+
if (!this.isConnected) return;
|
|
1401
|
+
await this.deps.admin.disconnect();
|
|
1402
|
+
this.isConnected = false;
|
|
1403
|
+
}
|
|
1404
|
+
async resetOffsets(groupId, topic2, position) {
|
|
1405
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1406
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1407
|
+
throw new Error(
|
|
1408
|
+
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
1409
|
+
);
|
|
1410
|
+
}
|
|
1411
|
+
await this.ensureConnected();
|
|
1412
|
+
const partitionOffsets = await this.deps.admin.fetchTopicOffsets(topic2);
|
|
1413
|
+
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
1414
|
+
partition,
|
|
1415
|
+
offset: position === "earliest" ? low : high
|
|
1416
|
+
}));
|
|
1417
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1418
|
+
this.deps.logger.log(
|
|
1419
|
+
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1420
|
+
);
|
|
1421
|
+
}
|
|
1422
|
+
/**
|
|
1423
|
+
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
1424
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
1425
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1426
|
+
*/
|
|
1427
|
+
async seekToOffset(groupId, assignments) {
|
|
1428
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1429
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1430
|
+
throw new Error(
|
|
1431
|
+
`seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1434
|
+
await this.ensureConnected();
|
|
1435
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
1436
|
+
for (const { topic: topic2, partition, offset } of assignments) {
|
|
1437
|
+
const list = byTopic.get(topic2) ?? [];
|
|
1438
|
+
list.push({ partition, offset });
|
|
1439
|
+
byTopic.set(topic2, list);
|
|
1440
|
+
}
|
|
1441
|
+
for (const [topic2, partitions] of byTopic) {
|
|
1442
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1443
|
+
this.deps.logger.log(
|
|
1444
|
+
`Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
|
|
1445
|
+
);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
/**
|
|
1449
|
+
* Seek specific topic-partition pairs to the offset nearest to a given timestamp
|
|
1450
|
+
* (in milliseconds) for a stopped consumer group.
|
|
1451
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
1452
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1453
|
+
* If no offset exists at the requested timestamp (e.g. empty partition or
|
|
1454
|
+
* future timestamp), the partition falls back to `-1` (end of topic — new messages only).
|
|
1455
|
+
*/
|
|
1456
|
+
async seekToTimestamp(groupId, assignments) {
|
|
1457
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1458
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
1459
|
+
throw new Error(
|
|
1460
|
+
`seekToTimestamp: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1461
|
+
);
|
|
1462
|
+
}
|
|
1463
|
+
await this.ensureConnected();
|
|
1464
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
1465
|
+
for (const { topic: topic2, partition, timestamp } of assignments) {
|
|
1466
|
+
const list = byTopic.get(topic2) ?? [];
|
|
1467
|
+
list.push({ partition, timestamp });
|
|
1468
|
+
byTopic.set(topic2, list);
|
|
1469
|
+
}
|
|
1470
|
+
for (const [topic2, parts] of byTopic) {
|
|
1471
|
+
const offsets = await Promise.all(
|
|
1472
|
+
parts.map(async ({ partition, timestamp }) => {
|
|
1473
|
+
const results = await this.deps.admin.fetchTopicOffsetsByTime(
|
|
1474
|
+
topic2,
|
|
1475
|
+
timestamp
|
|
1476
|
+
);
|
|
1477
|
+
const found = results.find(
|
|
1478
|
+
(r) => r.partition === partition
|
|
1479
|
+
);
|
|
1480
|
+
return { partition, offset: found?.offset ?? "-1" };
|
|
1481
|
+
})
|
|
1482
|
+
);
|
|
1483
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
|
|
1484
|
+
this.deps.logger.log(
|
|
1485
|
+
`Offsets set by timestamp for group "${gid}" on "${topic2}": ${JSON.stringify(offsets)}`
|
|
1486
|
+
);
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
/**
|
|
1490
|
+
* Query consumer group lag per partition.
|
|
1491
|
+
* Lag = broker high-watermark − last committed offset.
|
|
1492
|
+
* A committed offset of -1 (nothing committed yet) counts as full lag.
|
|
1493
|
+
*
|
|
1494
|
+
* Returns an empty array when the consumer group has never committed any
|
|
1495
|
+
* offsets (freshly created group, `autoCommit: false` with no manual commits,
|
|
1496
|
+
* or group not yet assigned). This is a Kafka protocol limitation:
|
|
1497
|
+
* `fetchOffsets` only returns data for topic-partitions that have at least one
|
|
1498
|
+
* committed offset. Use `checkStatus()` to verify broker connectivity in that case.
|
|
1499
|
+
*/
|
|
1500
|
+
async getConsumerLag(groupId) {
|
|
1501
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
1502
|
+
await this.ensureConnected();
|
|
1503
|
+
const committedByTopic = await this.deps.admin.fetchOffsets({ groupId: gid });
|
|
1504
|
+
const brokerOffsetsAll = await Promise.all(
|
|
1505
|
+
committedByTopic.map(({ topic: topic2 }) => this.deps.admin.fetchTopicOffsets(topic2))
|
|
1506
|
+
);
|
|
1507
|
+
const result = [];
|
|
1508
|
+
for (let i = 0; i < committedByTopic.length; i++) {
|
|
1509
|
+
const { topic: topic2, partitions } = committedByTopic[i];
|
|
1510
|
+
const brokerOffsets = brokerOffsetsAll[i];
|
|
1511
|
+
for (const { partition, offset } of partitions) {
|
|
1512
|
+
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
1513
|
+
if (!broker) continue;
|
|
1514
|
+
const committed = parseInt(offset, 10);
|
|
1515
|
+
const high = parseInt(broker.high, 10);
|
|
1516
|
+
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
1517
|
+
result.push({ topic: topic2, partition, lag });
|
|
1518
|
+
}
|
|
1519
|
+
}
|
|
1520
|
+
return result;
|
|
1521
|
+
}
|
|
1522
|
+
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
1523
|
+
async checkStatus() {
|
|
1524
|
+
try {
|
|
1525
|
+
await this.ensureConnected();
|
|
1526
|
+
const topics = await this.deps.admin.listTopics();
|
|
1527
|
+
return { status: "up", clientId: this.deps.clientId, topics };
|
|
1528
|
+
} catch (error) {
|
|
1529
|
+
return {
|
|
1530
|
+
status: "down",
|
|
1531
|
+
clientId: this.deps.clientId,
|
|
1532
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1533
|
+
};
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
/**
|
|
1537
|
+
* List all consumer groups known to the broker.
|
|
1538
|
+
* Useful for monitoring which groups are active and their current state.
|
|
1539
|
+
*/
|
|
1540
|
+
async listConsumerGroups() {
|
|
1541
|
+
await this.ensureConnected();
|
|
1542
|
+
const result = await this.deps.admin.listGroups();
|
|
1543
|
+
return result.groups.map((g) => ({
|
|
1544
|
+
groupId: g.groupId,
|
|
1545
|
+
state: g.state ?? "Unknown"
|
|
1546
|
+
}));
|
|
1547
|
+
}
|
|
1548
|
+
/**
|
|
1549
|
+
* Describe topics — returns partition layout, leader, replicas, and ISR.
|
|
1550
|
+
* @param topics Topic names to describe. Omit to describe all topics.
|
|
1551
|
+
*/
|
|
1552
|
+
async describeTopics(topics) {
|
|
1553
|
+
await this.ensureConnected();
|
|
1554
|
+
const result = await this.deps.admin.fetchTopicMetadata(
|
|
1555
|
+
topics ? { topics } : void 0
|
|
1556
|
+
);
|
|
1557
|
+
return result.topics.map((t) => ({
|
|
1558
|
+
name: t.name,
|
|
1559
|
+
partitions: t.partitions.map((p) => ({
|
|
1560
|
+
partition: p.partitionId ?? p.partition,
|
|
1561
|
+
leader: p.leader,
|
|
1562
|
+
replicas: p.replicas.map(
|
|
1563
|
+
(r) => typeof r === "number" ? r : r.nodeId
|
|
1564
|
+
),
|
|
1565
|
+
isr: p.isr.map(
|
|
1566
|
+
(r) => typeof r === "number" ? r : r.nodeId
|
|
1567
|
+
)
|
|
1568
|
+
}))
|
|
1569
|
+
}));
|
|
1570
|
+
}
|
|
1571
|
+
/**
|
|
1572
|
+
* Delete records from a topic up to (but not including) the given offsets.
|
|
1573
|
+
* All messages with offsets **before** the given offset are deleted.
|
|
1574
|
+
*/
|
|
1575
|
+
async deleteRecords(topic2, partitions) {
|
|
1576
|
+
await this.ensureConnected();
|
|
1577
|
+
await this.deps.admin.deleteTopicRecords({ topic: topic2, partitions });
|
|
1578
|
+
}
|
|
1579
|
+
/**
|
|
1580
|
+
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
1581
|
+
* `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
|
|
1582
|
+
* rather than silently discovering missing topics on the first handler failure.
|
|
1583
|
+
*/
|
|
1584
|
+
async validateRetryTopicsExist(topicNames, maxRetries) {
|
|
1585
|
+
await this.ensureConnected();
|
|
1586
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1587
|
+
const missing = [];
|
|
1588
|
+
for (const t of topicNames) {
|
|
1589
|
+
for (let level = 1; level <= maxRetries; level++) {
|
|
1590
|
+
const retryTopic = `${t}.retry.${level}`;
|
|
1591
|
+
if (!existing.has(retryTopic)) missing.push(retryTopic);
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
if (missing.length > 0) {
|
|
1595
|
+
throw new Error(
|
|
1596
|
+
`retryTopics: true but the following retry topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1597
|
+
);
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
/**
|
|
1601
|
+
* When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
|
|
1602
|
+
* consumed topic. Throws a clear error at startup rather than silently discovering
|
|
1603
|
+
* missing DLQ topics on the first handler failure.
|
|
1604
|
+
*/
|
|
1605
|
+
async validateDlqTopicsExist(topicNames) {
|
|
1606
|
+
await this.ensureConnected();
|
|
1607
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1608
|
+
const missing = topicNames.filter((t) => !existing.has(`${t}.dlq`)).map((t) => `${t}.dlq`);
|
|
1609
|
+
if (missing.length > 0) {
|
|
1610
|
+
throw new Error(
|
|
1611
|
+
`dlq: true but the following DLQ topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1612
|
+
);
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
/**
|
|
1616
|
+
* When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
|
|
1617
|
+
* that every `<topic>.duplicates` destination topic already exists. Throws a
|
|
1618
|
+
* clear error at startup rather than silently dropping duplicates on first hit.
|
|
1619
|
+
*/
|
|
1620
|
+
async validateDuplicatesTopicsExist(topicNames, customDestination) {
|
|
1621
|
+
await this.ensureConnected();
|
|
1622
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
1623
|
+
const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
|
|
1624
|
+
const missing = toCheck.filter((t) => !existing.has(t));
|
|
1625
|
+
if (missing.length > 0) {
|
|
1626
|
+
throw new Error(
|
|
1627
|
+
`deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
1628
|
+
);
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
};
|
|
1632
|
+
|
|
1633
|
+
// src/client/kafka.client/consumer/dlq-replay.ts
|
|
1634
|
+
async function replayDlqTopic(topic2, options = {}, deps) {
|
|
1635
|
+
const dlqTopic = `${topic2}.dlq`;
|
|
1636
|
+
const partitionOffsets = await deps.fetchTopicOffsets(dlqTopic);
|
|
1637
|
+
const activePartitions = partitionOffsets.filter((p) => parseInt(p.high, 10) > 0);
|
|
1638
|
+
if (activePartitions.length === 0) {
|
|
1639
|
+
deps.logger.log(`replayDlq: "${dlqTopic}" is empty \u2014 nothing to replay`);
|
|
1640
|
+
return { replayed: 0, skipped: 0 };
|
|
1641
|
+
}
|
|
1642
|
+
const highWatermarks = new Map(
|
|
1643
|
+
activePartitions.map(({ partition, high }) => [partition, parseInt(high, 10)])
|
|
1644
|
+
);
|
|
1645
|
+
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1646
|
+
let replayed = 0;
|
|
1647
|
+
let skipped = 0;
|
|
1648
|
+
const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
|
|
1649
|
+
await new Promise((resolve, reject) => {
|
|
1650
|
+
const consumer = deps.createConsumer(tempGroupId);
|
|
1651
|
+
const cleanup = () => deps.cleanupConsumer(consumer, tempGroupId);
|
|
1652
|
+
consumer.connect().then(() => subscribeWithRetry(consumer, [dlqTopic], deps.logger)).then(
|
|
1653
|
+
() => consumer.run({
|
|
1654
|
+
eachMessage: async ({ partition, message }) => {
|
|
1655
|
+
if (!message.value) return;
|
|
1656
|
+
const offset = parseInt(message.offset, 10);
|
|
1657
|
+
processedOffsets.set(partition, offset);
|
|
1658
|
+
const headers = decodeHeaders(message.headers);
|
|
1659
|
+
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1660
|
+
const originalHeaders = Object.fromEntries(
|
|
1661
|
+
Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
|
|
1662
|
+
);
|
|
1663
|
+
const value = message.value.toString();
|
|
1664
|
+
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1665
|
+
if (!targetTopic || !shouldProcess) {
|
|
1666
|
+
skipped++;
|
|
1667
|
+
} else if (options.dryRun) {
|
|
1668
|
+
deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
|
|
1669
|
+
replayed++;
|
|
1670
|
+
} else {
|
|
1671
|
+
await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
|
|
1672
|
+
replayed++;
|
|
1673
|
+
}
|
|
1674
|
+
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1675
|
+
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1676
|
+
);
|
|
1677
|
+
if (allDone) {
|
|
1678
|
+
cleanup();
|
|
1679
|
+
resolve();
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
})
|
|
1683
|
+
).catch((err) => {
|
|
1684
|
+
cleanup();
|
|
1685
|
+
reject(err);
|
|
1686
|
+
});
|
|
1687
|
+
});
|
|
1688
|
+
deps.logger.log(`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`);
|
|
1689
|
+
return { replayed, skipped };
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
// src/client/kafka.client/infra/metrics-manager.ts
|
|
1693
|
+
var MetricsManager = class {
|
|
1694
|
+
constructor(deps) {
|
|
1695
|
+
this.deps = deps;
|
|
1696
|
+
}
|
|
1697
|
+
topicMetrics = /* @__PURE__ */ new Map();
|
|
1698
|
+
metricsFor(topic2) {
|
|
1699
|
+
let m = this.topicMetrics.get(topic2);
|
|
1700
|
+
if (!m) {
|
|
1701
|
+
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1702
|
+
this.topicMetrics.set(topic2, m);
|
|
1703
|
+
}
|
|
1704
|
+
return m;
|
|
1705
|
+
}
|
|
1706
|
+
/** Fire `afterSend` instrumentation hooks for each message in a batch. */
|
|
1707
|
+
notifyAfterSend(topic2, count) {
|
|
1708
|
+
for (let i = 0; i < count; i++)
|
|
1709
|
+
for (const inst of this.deps.instrumentation) inst.afterSend?.(topic2);
|
|
1710
|
+
}
|
|
1711
|
+
notifyRetry(envelope, attempt, maxRetries) {
|
|
1712
|
+
this.metricsFor(envelope.topic).retryCount++;
|
|
1713
|
+
for (const inst of this.deps.instrumentation) inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1714
|
+
}
|
|
1715
|
+
notifyDlq(envelope, reason, gid) {
|
|
1716
|
+
this.metricsFor(envelope.topic).dlqCount++;
|
|
1717
|
+
for (const inst of this.deps.instrumentation) inst.onDlq?.(envelope, reason);
|
|
1718
|
+
if (gid) this.deps.onCircuitFailure(envelope, gid);
|
|
1719
|
+
}
|
|
1720
|
+
notifyDuplicate(envelope, strategy) {
|
|
1721
|
+
this.metricsFor(envelope.topic).dedupCount++;
|
|
1722
|
+
for (const inst of this.deps.instrumentation) inst.onDuplicate?.(envelope, strategy);
|
|
1723
|
+
}
|
|
1724
|
+
notifyMessage(envelope, gid) {
|
|
1725
|
+
this.metricsFor(envelope.topic).processedCount++;
|
|
1726
|
+
for (const inst of this.deps.instrumentation) inst.onMessage?.(envelope);
|
|
1727
|
+
if (gid) this.deps.onCircuitSuccess(envelope, gid);
|
|
1728
|
+
}
|
|
1729
|
+
getMetrics(topic2) {
|
|
1730
|
+
if (topic2 !== void 0) {
|
|
1731
|
+
const m = this.topicMetrics.get(topic2);
|
|
1732
|
+
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1733
|
+
}
|
|
1734
|
+
const agg = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1735
|
+
for (const m of this.topicMetrics.values()) {
|
|
1736
|
+
agg.processedCount += m.processedCount;
|
|
1737
|
+
agg.retryCount += m.retryCount;
|
|
1738
|
+
agg.dlqCount += m.dlqCount;
|
|
1739
|
+
agg.dedupCount += m.dedupCount;
|
|
1740
|
+
}
|
|
1741
|
+
return agg;
|
|
1742
|
+
}
|
|
1743
|
+
resetMetrics(topic2) {
|
|
1744
|
+
if (topic2 !== void 0) {
|
|
1745
|
+
this.topicMetrics.delete(topic2);
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1748
|
+
this.topicMetrics.clear();
|
|
1749
|
+
}
|
|
1750
|
+
};
|
|
1751
|
+
|
|
1752
|
+
// src/client/kafka.client/infra/inflight-tracker.ts
|
|
1753
|
+
var InFlightTracker = class {
|
|
1754
|
+
constructor(warn) {
|
|
1755
|
+
this.warn = warn;
|
|
1756
|
+
}
|
|
1757
|
+
inFlightTotal = 0;
|
|
1758
|
+
drainResolvers = [];
|
|
1759
|
+
track(fn) {
|
|
1760
|
+
this.inFlightTotal++;
|
|
1761
|
+
return fn().finally(() => {
|
|
1762
|
+
this.inFlightTotal--;
|
|
1763
|
+
if (this.inFlightTotal === 0) this.drainResolvers.splice(0).forEach((r) => r());
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1766
|
+
waitForDrain(timeoutMs) {
|
|
1767
|
+
if (this.inFlightTotal === 0) return Promise.resolve();
|
|
1768
|
+
return new Promise((resolve) => {
|
|
1769
|
+
let handle;
|
|
1770
|
+
const onDrain = () => {
|
|
1771
|
+
clearTimeout(handle);
|
|
1772
|
+
resolve();
|
|
1773
|
+
};
|
|
1774
|
+
this.drainResolvers.push(onDrain);
|
|
1775
|
+
handle = setTimeout(() => {
|
|
1776
|
+
const idx = this.drainResolvers.indexOf(onDrain);
|
|
1777
|
+
if (idx !== -1) this.drainResolvers.splice(idx, 1);
|
|
1778
|
+
this.warn(
|
|
1779
|
+
`Drain timed out after ${timeoutMs}ms \u2014 ${this.inFlightTotal} handler(s) still in flight`
|
|
1780
|
+
);
|
|
1781
|
+
resolve();
|
|
1782
|
+
}, timeoutMs);
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
};
|
|
1786
|
+
|
|
1787
|
+
// src/client/kafka.client/index.ts
|
|
1788
|
+
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS2;
|
|
1789
|
+
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1236
1790
|
var KafkaClient = class _KafkaClient {
|
|
1237
1791
|
kafka;
|
|
1238
1792
|
producer;
|
|
@@ -1241,7 +1795,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1241
1795
|
/** Maps transactionalId → Producer for each active retry level consumer. */
|
|
1242
1796
|
retryTxProducers = /* @__PURE__ */ new Map();
|
|
1243
1797
|
consumers = /* @__PURE__ */ new Map();
|
|
1244
|
-
admin;
|
|
1245
1798
|
logger;
|
|
1246
1799
|
autoCreateTopicsEnabled;
|
|
1247
1800
|
strictSchemasEnabled;
|
|
@@ -1261,20 +1814,26 @@ var KafkaClient = class _KafkaClient {
|
|
|
1261
1814
|
onRebalance;
|
|
1262
1815
|
/** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
|
|
1263
1816
|
txId;
|
|
1264
|
-
/** Per-topic event counters, lazily created on first event. Aggregated by `getMetrics()`. */
|
|
1265
|
-
_topicMetrics = /* @__PURE__ */ new Map();
|
|
1266
1817
|
/** Monotonically increasing Lamport clock stamped on every outgoing message. */
|
|
1267
1818
|
_lamportClock = 0;
|
|
1268
1819
|
/** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
|
|
1269
1820
|
dedupStates = /* @__PURE__ */ new Map();
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
isAdminConnected = false;
|
|
1275
|
-
inFlightTotal = 0;
|
|
1276
|
-
drainResolvers = [];
|
|
1821
|
+
circuitBreaker;
|
|
1822
|
+
adminOps;
|
|
1823
|
+
metrics;
|
|
1824
|
+
inFlight;
|
|
1277
1825
|
clientId;
|
|
1826
|
+
_producerOpsDeps;
|
|
1827
|
+
_consumerOpsDeps;
|
|
1828
|
+
_retryTopicDeps;
|
|
1829
|
+
/** DLQ header keys added by the pipeline — stripped before re-publishing. */
|
|
1830
|
+
static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
1831
|
+
"x-dlq-original-topic",
|
|
1832
|
+
"x-dlq-failed-at",
|
|
1833
|
+
"x-dlq-error-message",
|
|
1834
|
+
"x-dlq-error-stack",
|
|
1835
|
+
"x-dlq-attempt-count"
|
|
1836
|
+
]);
|
|
1278
1837
|
constructor(clientId, groupId, brokers, options) {
|
|
1279
1838
|
this.clientId = clientId;
|
|
1280
1839
|
this.defaultGroupId = groupId;
|
|
@@ -1299,12 +1858,41 @@ var KafkaClient = class _KafkaClient {
|
|
|
1299
1858
|
logLevel: KafkaLogLevel.ERROR
|
|
1300
1859
|
}
|
|
1301
1860
|
});
|
|
1302
|
-
this.producer = this.kafka.producer({
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1861
|
+
this.producer = this.kafka.producer({ kafkaJS: { acks: -1 } });
|
|
1862
|
+
this.adminOps = new AdminOps({
|
|
1863
|
+
admin: this.kafka.admin(),
|
|
1864
|
+
logger: this.logger,
|
|
1865
|
+
runningConsumers: this.runningConsumers,
|
|
1866
|
+
defaultGroupId: this.defaultGroupId,
|
|
1867
|
+
clientId: this.clientId
|
|
1868
|
+
});
|
|
1869
|
+
this.circuitBreaker = new CircuitBreakerManager({
|
|
1870
|
+
pauseConsumer: (gid, assignments) => this.pauseConsumer(gid, assignments),
|
|
1871
|
+
resumeConsumer: (gid, assignments) => this.resumeConsumer(gid, assignments),
|
|
1872
|
+
logger: this.logger,
|
|
1873
|
+
instrumentation: this.instrumentation
|
|
1874
|
+
});
|
|
1875
|
+
this.metrics = new MetricsManager({
|
|
1876
|
+
instrumentation: this.instrumentation,
|
|
1877
|
+
onCircuitFailure: (envelope, gid) => this.circuitBreaker.onFailure(envelope, gid),
|
|
1878
|
+
onCircuitSuccess: (envelope, gid) => this.circuitBreaker.onSuccess(envelope, gid)
|
|
1306
1879
|
});
|
|
1307
|
-
this.
|
|
1880
|
+
this.inFlight = new InFlightTracker((msg) => this.logger.warn(msg));
|
|
1881
|
+
this._producerOpsDeps = {
|
|
1882
|
+
schemaRegistry: this.schemaRegistry,
|
|
1883
|
+
strictSchemasEnabled: this.strictSchemasEnabled,
|
|
1884
|
+
instrumentation: this.instrumentation,
|
|
1885
|
+
logger: this.logger,
|
|
1886
|
+
nextLamportClock: () => ++this._lamportClock
|
|
1887
|
+
};
|
|
1888
|
+
this._consumerOpsDeps = {
|
|
1889
|
+
consumers: this.consumers,
|
|
1890
|
+
consumerCreationOptions: this.consumerCreationOptions,
|
|
1891
|
+
kafka: this.kafka,
|
|
1892
|
+
onRebalance: this.onRebalance,
|
|
1893
|
+
logger: this.logger
|
|
1894
|
+
};
|
|
1895
|
+
this._retryTopicDeps = this.buildRetryTopicDeps();
|
|
1308
1896
|
}
|
|
1309
1897
|
async sendMessage(topicOrDesc, message, options = {}) {
|
|
1310
1898
|
const payload = await this.preparePayload(
|
|
@@ -1322,7 +1910,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1322
1910
|
options.compression
|
|
1323
1911
|
);
|
|
1324
1912
|
await this.producer.send(payload);
|
|
1325
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1913
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1326
1914
|
}
|
|
1327
1915
|
/**
|
|
1328
1916
|
* Send a null-value (tombstone) message. Used with log-compacted topics to signal
|
|
@@ -1337,26 +1925,15 @@ var KafkaClient = class _KafkaClient {
|
|
|
1337
1925
|
*/
|
|
1338
1926
|
async sendTombstone(topic2, key, headers) {
|
|
1339
1927
|
const hdrs = { ...headers };
|
|
1340
|
-
for (const inst of this.instrumentation)
|
|
1341
|
-
inst.beforeSend?.(topic2, hdrs);
|
|
1342
|
-
}
|
|
1928
|
+
for (const inst of this.instrumentation) inst.beforeSend?.(topic2, hdrs);
|
|
1343
1929
|
await this.ensureTopic(topic2);
|
|
1344
|
-
await this.producer.send({
|
|
1345
|
-
|
|
1346
|
-
messages: [{ value: null, key, headers: hdrs }]
|
|
1347
|
-
});
|
|
1348
|
-
for (const inst of this.instrumentation) {
|
|
1349
|
-
inst.afterSend?.(topic2);
|
|
1350
|
-
}
|
|
1930
|
+
await this.producer.send({ topic: topic2, messages: [{ value: null, key, headers: hdrs }] });
|
|
1931
|
+
for (const inst of this.instrumentation) inst.afterSend?.(topic2);
|
|
1351
1932
|
}
|
|
1352
1933
|
async sendBatch(topicOrDesc, messages, options) {
|
|
1353
|
-
const payload = await this.preparePayload(
|
|
1354
|
-
topicOrDesc,
|
|
1355
|
-
messages,
|
|
1356
|
-
options?.compression
|
|
1357
|
-
);
|
|
1934
|
+
const payload = await this.preparePayload(topicOrDesc, messages, options?.compression);
|
|
1358
1935
|
await this.producer.send(payload);
|
|
1359
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1936
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1360
1937
|
}
|
|
1361
1938
|
/** Execute multiple sends atomically. Commits on success, aborts on error. */
|
|
1362
1939
|
async transaction(fn) {
|
|
@@ -1368,12 +1945,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1368
1945
|
}
|
|
1369
1946
|
const initPromise = (async () => {
|
|
1370
1947
|
const p = this.kafka.producer({
|
|
1371
|
-
kafkaJS: {
|
|
1372
|
-
acks: -1,
|
|
1373
|
-
idempotent: true,
|
|
1374
|
-
transactionalId: this.txId,
|
|
1375
|
-
maxInFlightRequests: 1
|
|
1376
|
-
}
|
|
1948
|
+
kafkaJS: { acks: -1, idempotent: true, transactionalId: this.txId, maxInFlightRequests: 1 }
|
|
1377
1949
|
});
|
|
1378
1950
|
await p.connect();
|
|
1379
1951
|
_activeTransactionalIds.add(this.txId);
|
|
@@ -1400,19 +1972,12 @@ var KafkaClient = class _KafkaClient {
|
|
|
1400
1972
|
}
|
|
1401
1973
|
]);
|
|
1402
1974
|
await tx.send(payload);
|
|
1403
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1975
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1404
1976
|
},
|
|
1405
|
-
/**
|
|
1406
|
-
* Send multiple messages in a single call to the topic.
|
|
1407
|
-
* All messages in the batch will be sent atomically.
|
|
1408
|
-
* If any message fails to send, the entire batch will be aborted.
|
|
1409
|
-
* @param topicOrDesc - topic name or TopicDescriptor
|
|
1410
|
-
* @param messages - array of messages to send with optional key, headers, correlationId, schemaVersion, and eventId
|
|
1411
|
-
*/
|
|
1412
1977
|
sendBatch: async (topicOrDesc, messages) => {
|
|
1413
1978
|
const payload = await this.preparePayload(topicOrDesc, messages);
|
|
1414
1979
|
await tx.send(payload);
|
|
1415
|
-
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1980
|
+
this.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1416
1981
|
}
|
|
1417
1982
|
};
|
|
1418
1983
|
await fn(ctx);
|
|
@@ -1421,10 +1986,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1421
1986
|
try {
|
|
1422
1987
|
await tx.abort();
|
|
1423
1988
|
} catch (abortError) {
|
|
1424
|
-
this.logger.error(
|
|
1425
|
-
"Failed to abort transaction:",
|
|
1426
|
-
toError(abortError).message
|
|
1427
|
-
);
|
|
1989
|
+
this.logger.error("Failed to abort transaction:", toError(abortError).message);
|
|
1428
1990
|
}
|
|
1429
1991
|
throw error;
|
|
1430
1992
|
}
|
|
@@ -1446,35 +2008,14 @@ var KafkaClient = class _KafkaClient {
|
|
|
1446
2008
|
this.logger.log("Producer disconnected");
|
|
1447
2009
|
}
|
|
1448
2010
|
async startConsumer(topics, handleMessage, options = {}) {
|
|
1449
|
-
|
|
1450
|
-
throw new Error(
|
|
1451
|
-
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1452
|
-
);
|
|
1453
|
-
}
|
|
1454
|
-
const hasRegexTopics = topics.some((t) => t instanceof RegExp);
|
|
1455
|
-
if (options.retryTopics && hasRegexTopics) {
|
|
1456
|
-
throw new Error(
|
|
1457
|
-
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
1458
|
-
);
|
|
1459
|
-
}
|
|
2011
|
+
this.validateTopicConsumerOpts(topics, options);
|
|
1460
2012
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1461
2013
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", setupOptions);
|
|
1462
|
-
if (options.circuitBreaker)
|
|
1463
|
-
this.circuitConfigs.set(gid, options.circuitBreaker);
|
|
2014
|
+
if (options.circuitBreaker) this.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
1464
2015
|
const deps = this.messageDepsFor(gid);
|
|
1465
|
-
const
|
|
1466
|
-
const deduplication = this.resolveDeduplicationContext(
|
|
1467
|
-
gid,
|
|
1468
|
-
options.deduplication
|
|
1469
|
-
);
|
|
1470
|
-
let eosMainContext;
|
|
1471
|
-
if (options.retryTopics && retry) {
|
|
1472
|
-
const mainTxId = `${gid}-main-tx`;
|
|
1473
|
-
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1474
|
-
eosMainContext = { txProducer, consumer };
|
|
1475
|
-
}
|
|
2016
|
+
const eosMainContext = await this.makeEosMainContext(gid, consumer, options);
|
|
1476
2017
|
await consumer.run({
|
|
1477
|
-
eachMessage: (payload) => this.
|
|
2018
|
+
eachMessage: (payload) => this.inFlight.track(
|
|
1478
2019
|
() => handleEachMessage(
|
|
1479
2020
|
payload,
|
|
1480
2021
|
{
|
|
@@ -1484,9 +2025,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1484
2025
|
dlq,
|
|
1485
2026
|
retry,
|
|
1486
2027
|
retryTopics: options.retryTopics,
|
|
1487
|
-
timeoutMs,
|
|
2028
|
+
timeoutMs: options.handlerTimeoutMs,
|
|
1488
2029
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1489
|
-
deduplication,
|
|
2030
|
+
deduplication: this.resolveDeduplicationContext(gid, options.deduplication),
|
|
1490
2031
|
messageTtlMs: options.messageTtlMs,
|
|
1491
2032
|
onTtlExpired: options.onTtlExpired,
|
|
1492
2033
|
eosMainContext
|
|
@@ -1497,70 +2038,24 @@ var KafkaClient = class _KafkaClient {
|
|
|
1497
2038
|
});
|
|
1498
2039
|
this.runningConsumers.set(gid, "eachMessage");
|
|
1499
2040
|
if (options.retryTopics && retry) {
|
|
1500
|
-
|
|
1501
|
-
await this.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
1502
|
-
}
|
|
1503
|
-
const companions = await startRetryTopicConsumers(
|
|
1504
|
-
topicNames,
|
|
1505
|
-
gid,
|
|
1506
|
-
handleMessage,
|
|
1507
|
-
retry,
|
|
1508
|
-
dlq,
|
|
1509
|
-
interceptors,
|
|
1510
|
-
schemaMap,
|
|
1511
|
-
this.retryTopicDeps,
|
|
1512
|
-
options.retryTopicAssignmentTimeoutMs
|
|
1513
|
-
);
|
|
1514
|
-
this.companionGroupIds.set(gid, companions);
|
|
2041
|
+
await this.launchRetryChain(gid, topicNames, handleMessage, retry, dlq, interceptors, schemaMap, options.retryTopicAssignmentTimeoutMs);
|
|
1515
2042
|
}
|
|
1516
2043
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
1517
2044
|
}
|
|
1518
2045
|
async startBatchConsumer(topics, handleBatch, options = {}) {
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1522
|
-
);
|
|
1523
|
-
}
|
|
1524
|
-
const hasRegexTopics = topics.some((t) => t instanceof RegExp);
|
|
1525
|
-
if (options.retryTopics && hasRegexTopics) {
|
|
1526
|
-
throw new Error(
|
|
1527
|
-
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
1528
|
-
);
|
|
1529
|
-
}
|
|
1530
|
-
if (options.retryTopics) {
|
|
1531
|
-
} else if (options.autoCommit !== false) {
|
|
2046
|
+
this.validateTopicConsumerOpts(topics, options);
|
|
2047
|
+
if (!options.retryTopics && options.autoCommit !== false) {
|
|
1532
2048
|
this.logger.debug?.(
|
|
1533
2049
|
`startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
|
|
1534
2050
|
);
|
|
1535
2051
|
}
|
|
1536
2052
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1537
2053
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", setupOptions);
|
|
1538
|
-
if (options.circuitBreaker)
|
|
1539
|
-
this.circuitConfigs.set(gid, options.circuitBreaker);
|
|
2054
|
+
if (options.circuitBreaker) this.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
1540
2055
|
const deps = this.messageDepsFor(gid);
|
|
1541
|
-
const
|
|
1542
|
-
const deduplication = this.resolveDeduplicationContext(
|
|
1543
|
-
gid,
|
|
1544
|
-
options.deduplication
|
|
1545
|
-
);
|
|
1546
|
-
let eosMainContext;
|
|
1547
|
-
if (options.retryTopics && retry) {
|
|
1548
|
-
const mainTxId = `${gid}-main-tx`;
|
|
1549
|
-
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1550
|
-
eosMainContext = { txProducer, consumer };
|
|
1551
|
-
}
|
|
2056
|
+
const eosMainContext = await this.makeEosMainContext(gid, consumer, options);
|
|
1552
2057
|
await consumer.run({
|
|
1553
|
-
|
|
1554
|
-
* eachBatch: called by the consumer for each batch of messages.
|
|
1555
|
-
* Called with the `payload` argument, which is an object containing the
|
|
1556
|
-
* batch of messages and a `BatchMeta` object with offset management controls.
|
|
1557
|
-
*
|
|
1558
|
-
* The function is wrapped with `trackInFlight` and `handleEachBatch` to provide
|
|
1559
|
-
* error handling and offset management.
|
|
1560
|
-
*
|
|
1561
|
-
* @param payload - an object containing the batch of messages and a `BatchMeta` object.
|
|
1562
|
-
*/
|
|
1563
|
-
eachBatch: (payload) => this.trackInFlight(
|
|
2058
|
+
eachBatch: (payload) => this.inFlight.track(
|
|
1564
2059
|
() => handleEachBatch(
|
|
1565
2060
|
payload,
|
|
1566
2061
|
{
|
|
@@ -1570,9 +2065,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1570
2065
|
dlq,
|
|
1571
2066
|
retry,
|
|
1572
2067
|
retryTopics: options.retryTopics,
|
|
1573
|
-
timeoutMs,
|
|
2068
|
+
timeoutMs: options.handlerTimeoutMs,
|
|
1574
2069
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1575
|
-
deduplication,
|
|
2070
|
+
deduplication: this.resolveDeduplicationContext(gid, options.deduplication),
|
|
1576
2071
|
messageTtlMs: options.messageTtlMs,
|
|
1577
2072
|
onTtlExpired: options.onTtlExpired,
|
|
1578
2073
|
eosMainContext
|
|
@@ -1583,9 +2078,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1583
2078
|
});
|
|
1584
2079
|
this.runningConsumers.set(gid, "eachBatch");
|
|
1585
2080
|
if (options.retryTopics && retry) {
|
|
1586
|
-
if (!this.autoCreateTopicsEnabled) {
|
|
1587
|
-
await this.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
1588
|
-
}
|
|
1589
2081
|
const handleMessageForRetry = (env) => handleBatch([env], {
|
|
1590
2082
|
partition: env.partition,
|
|
1591
2083
|
highWatermark: null,
|
|
@@ -1596,18 +2088,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1596
2088
|
commitOffsetsIfNecessary: async () => {
|
|
1597
2089
|
}
|
|
1598
2090
|
});
|
|
1599
|
-
|
|
1600
|
-
topicNames,
|
|
1601
|
-
gid,
|
|
1602
|
-
handleMessageForRetry,
|
|
1603
|
-
retry,
|
|
1604
|
-
dlq,
|
|
1605
|
-
interceptors,
|
|
1606
|
-
schemaMap,
|
|
1607
|
-
this.retryTopicDeps,
|
|
1608
|
-
options.retryTopicAssignmentTimeoutMs
|
|
1609
|
-
);
|
|
1610
|
-
this.companionGroupIds.set(gid, companions);
|
|
2091
|
+
await this.launchRetryChain(gid, topicNames, handleMessageForRetry, retry, dlq, interceptors, schemaMap, options.retryTopicAssignmentTimeoutMs);
|
|
1611
2092
|
}
|
|
1612
2093
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
1613
2094
|
}
|
|
@@ -1666,38 +2147,20 @@ var KafkaClient = class _KafkaClient {
|
|
|
1666
2147
|
if (groupId !== void 0) {
|
|
1667
2148
|
const consumer = this.consumers.get(groupId);
|
|
1668
2149
|
if (!consumer) {
|
|
1669
|
-
this.logger.warn(
|
|
1670
|
-
`stopConsumer: no active consumer for group "${groupId}"`
|
|
1671
|
-
);
|
|
2150
|
+
this.logger.warn(`stopConsumer: no active consumer for group "${groupId}"`);
|
|
1672
2151
|
return;
|
|
1673
2152
|
}
|
|
1674
|
-
await consumer.disconnect().catch(
|
|
1675
|
-
(e) => this.logger.warn(
|
|
1676
|
-
`Error disconnecting consumer "${groupId}":`,
|
|
1677
|
-
toError(e).message
|
|
1678
|
-
)
|
|
1679
|
-
);
|
|
2153
|
+
await consumer.disconnect().catch((e) => this.logger.warn(`Error disconnecting consumer "${groupId}":`, toError(e).message));
|
|
1680
2154
|
this.consumers.delete(groupId);
|
|
1681
2155
|
this.runningConsumers.delete(groupId);
|
|
1682
2156
|
this.consumerCreationOptions.delete(groupId);
|
|
1683
2157
|
this.dedupStates.delete(groupId);
|
|
1684
|
-
|
|
1685
|
-
if (key.startsWith(`${groupId}:`)) {
|
|
1686
|
-
clearTimeout(this.circuitStates.get(key).timer);
|
|
1687
|
-
this.circuitStates.delete(key);
|
|
1688
|
-
}
|
|
1689
|
-
}
|
|
1690
|
-
this.circuitConfigs.delete(groupId);
|
|
2158
|
+
this.circuitBreaker.removeGroup(groupId);
|
|
1691
2159
|
this.logger.log(`Consumer disconnected: group "${groupId}"`);
|
|
1692
2160
|
const mainTxId = `${groupId}-main-tx`;
|
|
1693
2161
|
const mainTxProducer = this.retryTxProducers.get(mainTxId);
|
|
1694
2162
|
if (mainTxProducer) {
|
|
1695
|
-
await mainTxProducer.disconnect().catch(
|
|
1696
|
-
(e) => this.logger.warn(
|
|
1697
|
-
`Error disconnecting main tx producer "${mainTxId}":`,
|
|
1698
|
-
toError(e).message
|
|
1699
|
-
)
|
|
1700
|
-
);
|
|
2163
|
+
await mainTxProducer.disconnect().catch((e) => this.logger.warn(`Error disconnecting main tx producer "${mainTxId}":`, toError(e).message));
|
|
1701
2164
|
_activeTransactionalIds.delete(mainTxId);
|
|
1702
2165
|
this.retryTxProducers.delete(mainTxId);
|
|
1703
2166
|
}
|
|
@@ -1705,12 +2168,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1705
2168
|
for (const cGroupId of companions) {
|
|
1706
2169
|
const cConsumer = this.consumers.get(cGroupId);
|
|
1707
2170
|
if (cConsumer) {
|
|
1708
|
-
await cConsumer.disconnect().catch(
|
|
1709
|
-
(e) => this.logger.warn(
|
|
1710
|
-
`Error disconnecting retry consumer "${cGroupId}":`,
|
|
1711
|
-
toError(e).message
|
|
1712
|
-
)
|
|
1713
|
-
);
|
|
2171
|
+
await cConsumer.disconnect().catch((e) => this.logger.warn(`Error disconnecting retry consumer "${cGroupId}":`, toError(e).message));
|
|
1714
2172
|
this.consumers.delete(cGroupId);
|
|
1715
2173
|
this.runningConsumers.delete(cGroupId);
|
|
1716
2174
|
this.consumerCreationOptions.delete(cGroupId);
|
|
@@ -1719,12 +2177,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1719
2177
|
const txId = `${cGroupId}-tx`;
|
|
1720
2178
|
const txProducer = this.retryTxProducers.get(txId);
|
|
1721
2179
|
if (txProducer) {
|
|
1722
|
-
await txProducer.disconnect().catch(
|
|
1723
|
-
(e) => this.logger.warn(
|
|
1724
|
-
`Error disconnecting retry tx producer "${txId}":`,
|
|
1725
|
-
toError(e).message
|
|
1726
|
-
)
|
|
1727
|
-
);
|
|
2180
|
+
await txProducer.disconnect().catch((e) => this.logger.warn(`Error disconnecting retry tx producer "${txId}":`, toError(e).message));
|
|
1728
2181
|
_activeTransactionalIds.delete(txId);
|
|
1729
2182
|
this.retryTxProducers.delete(txId);
|
|
1730
2183
|
}
|
|
@@ -1732,14 +2185,10 @@ var KafkaClient = class _KafkaClient {
|
|
|
1732
2185
|
this.companionGroupIds.delete(groupId);
|
|
1733
2186
|
} else {
|
|
1734
2187
|
const tasks = [
|
|
1735
|
-
...Array.from(this.consumers.values()).map(
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
)
|
|
1739
|
-
...Array.from(this.retryTxProducers.values()).map(
|
|
1740
|
-
(p) => p.disconnect().catch(() => {
|
|
1741
|
-
})
|
|
1742
|
-
)
|
|
2188
|
+
...Array.from(this.consumers.values()).map((c) => c.disconnect().catch(() => {
|
|
2189
|
+
})),
|
|
2190
|
+
...Array.from(this.retryTxProducers.values()).map((p) => p.disconnect().catch(() => {
|
|
2191
|
+
}))
|
|
1743
2192
|
];
|
|
1744
2193
|
await Promise.allSettled(tasks);
|
|
1745
2194
|
this.consumers.clear();
|
|
@@ -1748,10 +2197,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1748
2197
|
this.companionGroupIds.clear();
|
|
1749
2198
|
this.retryTxProducers.clear();
|
|
1750
2199
|
this.dedupStates.clear();
|
|
1751
|
-
|
|
1752
|
-
clearTimeout(state.timer);
|
|
1753
|
-
this.circuitStates.clear();
|
|
1754
|
-
this.circuitConfigs.clear();
|
|
2200
|
+
this.circuitBreaker.clear();
|
|
1755
2201
|
this.logger.log("All consumers disconnected");
|
|
1756
2202
|
}
|
|
1757
2203
|
}
|
|
@@ -1769,9 +2215,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1769
2215
|
return;
|
|
1770
2216
|
}
|
|
1771
2217
|
consumer.pause(
|
|
1772
|
-
assignments.flatMap(
|
|
1773
|
-
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1774
|
-
)
|
|
2218
|
+
assignments.flatMap(({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] })))
|
|
1775
2219
|
);
|
|
1776
2220
|
}
|
|
1777
2221
|
/**
|
|
@@ -1788,9 +2232,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1788
2232
|
return;
|
|
1789
2233
|
}
|
|
1790
2234
|
consumer.resume(
|
|
1791
|
-
assignments.flatMap(
|
|
1792
|
-
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1793
|
-
)
|
|
2235
|
+
assignments.flatMap(({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] })))
|
|
1794
2236
|
);
|
|
1795
2237
|
}
|
|
1796
2238
|
/** Pause all assigned partitions of a topic for a consumer group (used for queue backpressure). */
|
|
@@ -1811,14 +2253,6 @@ var KafkaClient = class _KafkaClient {
|
|
|
1811
2253
|
if (partitions.length > 0)
|
|
1812
2254
|
consumer.resume(partitions.map((p) => ({ topic: topic2, partitions: [p] })));
|
|
1813
2255
|
}
|
|
1814
|
-
/** DLQ header keys added by `sendToDlq` — stripped before re-publishing. */
|
|
1815
|
-
static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
1816
|
-
"x-dlq-original-topic",
|
|
1817
|
-
"x-dlq-failed-at",
|
|
1818
|
-
"x-dlq-error-message",
|
|
1819
|
-
"x-dlq-error-stack",
|
|
1820
|
-
"x-dlq-attempt-count"
|
|
1821
|
-
]);
|
|
1822
2256
|
/**
|
|
1823
2257
|
* Re-publish messages from a dead letter queue back to the original topic.
|
|
1824
2258
|
*
|
|
@@ -1831,106 +2265,27 @@ var KafkaClient = class _KafkaClient {
|
|
|
1831
2265
|
* @returns { replayed: number; skipped: number } - counts of re-published vs skipped messages
|
|
1832
2266
|
*/
|
|
1833
2267
|
async replayDlq(topic2, options = {}) {
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
}
|
|
1844
|
-
const highWatermarks = new Map(
|
|
1845
|
-
activePartitions.map(({ partition, high }) => [
|
|
1846
|
-
partition,
|
|
1847
|
-
parseInt(high, 10)
|
|
1848
|
-
])
|
|
1849
|
-
);
|
|
1850
|
-
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1851
|
-
let replayed = 0;
|
|
1852
|
-
let skipped = 0;
|
|
1853
|
-
const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
|
|
1854
|
-
await new Promise((resolve, reject) => {
|
|
1855
|
-
const consumer = getOrCreateConsumer(
|
|
1856
|
-
tempGroupId,
|
|
1857
|
-
true,
|
|
1858
|
-
true,
|
|
1859
|
-
this.consumerOpsDeps
|
|
1860
|
-
);
|
|
1861
|
-
const cleanup = () => {
|
|
2268
|
+
await this.adminOps.ensureConnected();
|
|
2269
|
+
return replayDlqTopic(topic2, options, {
|
|
2270
|
+
logger: this.logger,
|
|
2271
|
+
fetchTopicOffsets: (t) => this.adminOps.admin.fetchTopicOffsets(t),
|
|
2272
|
+
send: async (t, messages) => {
|
|
2273
|
+
await this.producer.send({ topic: t, messages });
|
|
2274
|
+
},
|
|
2275
|
+
createConsumer: (gid) => getOrCreateConsumer(gid, true, true, this._consumerOpsDeps),
|
|
2276
|
+
cleanupConsumer: (consumer, gid) => {
|
|
1862
2277
|
consumer.disconnect().catch(() => {
|
|
1863
2278
|
}).finally(() => {
|
|
1864
|
-
this.consumers.delete(
|
|
1865
|
-
this.runningConsumers.delete(
|
|
1866
|
-
this.consumerCreationOptions.delete(
|
|
2279
|
+
this.consumers.delete(gid);
|
|
2280
|
+
this.runningConsumers.delete(gid);
|
|
2281
|
+
this.consumerCreationOptions.delete(gid);
|
|
1867
2282
|
});
|
|
1868
|
-
}
|
|
1869
|
-
|
|
1870
|
-
() => consumer.run({
|
|
1871
|
-
eachMessage: async ({ partition, message }) => {
|
|
1872
|
-
if (!message.value) return;
|
|
1873
|
-
const offset = parseInt(message.offset, 10);
|
|
1874
|
-
processedOffsets.set(partition, offset);
|
|
1875
|
-
const headers = decodeHeaders(message.headers);
|
|
1876
|
-
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1877
|
-
const originalHeaders = Object.fromEntries(
|
|
1878
|
-
Object.entries(headers).filter(
|
|
1879
|
-
([k]) => !_KafkaClient.DLQ_HEADER_KEYS.has(k)
|
|
1880
|
-
)
|
|
1881
|
-
);
|
|
1882
|
-
const value = message.value.toString();
|
|
1883
|
-
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1884
|
-
if (!targetTopic || !shouldProcess) {
|
|
1885
|
-
skipped++;
|
|
1886
|
-
} else if (options.dryRun) {
|
|
1887
|
-
this.logger.log(
|
|
1888
|
-
`[DLQ replay dry-run] Would replay to "${targetTopic}"`
|
|
1889
|
-
);
|
|
1890
|
-
replayed++;
|
|
1891
|
-
} else {
|
|
1892
|
-
await this.producer.send({
|
|
1893
|
-
topic: targetTopic,
|
|
1894
|
-
messages: [{ value, headers: originalHeaders }]
|
|
1895
|
-
});
|
|
1896
|
-
replayed++;
|
|
1897
|
-
}
|
|
1898
|
-
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1899
|
-
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1900
|
-
);
|
|
1901
|
-
if (allDone) {
|
|
1902
|
-
cleanup();
|
|
1903
|
-
resolve();
|
|
1904
|
-
}
|
|
1905
|
-
}
|
|
1906
|
-
})
|
|
1907
|
-
).catch((err) => {
|
|
1908
|
-
cleanup();
|
|
1909
|
-
reject(err);
|
|
1910
|
-
});
|
|
2283
|
+
},
|
|
2284
|
+
dlqHeaderKeys: _KafkaClient.DLQ_HEADER_KEYS
|
|
1911
2285
|
});
|
|
1912
|
-
this.logger.log(
|
|
1913
|
-
`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`
|
|
1914
|
-
);
|
|
1915
|
-
return { replayed, skipped };
|
|
1916
2286
|
}
|
|
1917
2287
|
async resetOffsets(groupId, topic2, position) {
|
|
1918
|
-
|
|
1919
|
-
if (this.runningConsumers.has(gid)) {
|
|
1920
|
-
throw new Error(
|
|
1921
|
-
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
1922
|
-
);
|
|
1923
|
-
}
|
|
1924
|
-
await this.ensureAdminConnected();
|
|
1925
|
-
const partitionOffsets = await this.admin.fetchTopicOffsets(topic2);
|
|
1926
|
-
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
1927
|
-
partition,
|
|
1928
|
-
offset: position === "earliest" ? low : high
|
|
1929
|
-
}));
|
|
1930
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1931
|
-
this.logger.log(
|
|
1932
|
-
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1933
|
-
);
|
|
2288
|
+
return this.adminOps.resetOffsets(groupId, topic2, position);
|
|
1934
2289
|
}
|
|
1935
2290
|
/**
|
|
1936
2291
|
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
@@ -1938,25 +2293,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1938
2293
|
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1939
2294
|
*/
|
|
1940
2295
|
async seekToOffset(groupId, assignments) {
|
|
1941
|
-
|
|
1942
|
-
if (this.runningConsumers.has(gid)) {
|
|
1943
|
-
throw new Error(
|
|
1944
|
-
`seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1945
|
-
);
|
|
1946
|
-
}
|
|
1947
|
-
await this.ensureAdminConnected();
|
|
1948
|
-
const byTopic = /* @__PURE__ */ new Map();
|
|
1949
|
-
for (const { topic: topic2, partition, offset } of assignments) {
|
|
1950
|
-
const list = byTopic.get(topic2) ?? [];
|
|
1951
|
-
list.push({ partition, offset });
|
|
1952
|
-
byTopic.set(topic2, list);
|
|
1953
|
-
}
|
|
1954
|
-
for (const [topic2, partitions] of byTopic) {
|
|
1955
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1956
|
-
this.logger.log(
|
|
1957
|
-
`Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
|
|
1958
|
-
);
|
|
1959
|
-
}
|
|
2296
|
+
return this.adminOps.seekToOffset(groupId, assignments);
|
|
1960
2297
|
}
|
|
1961
2298
|
/**
|
|
1962
2299
|
* Seek specific topic-partition pairs to the offset nearest to a given timestamp
|
|
@@ -1967,37 +2304,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1967
2304
|
* future timestamp), the partition falls back to `-1` (end of topic — new messages only).
|
|
1968
2305
|
*/
|
|
1969
2306
|
async seekToTimestamp(groupId, assignments) {
|
|
1970
|
-
|
|
1971
|
-
if (this.runningConsumers.has(gid)) {
|
|
1972
|
-
throw new Error(
|
|
1973
|
-
`seekToTimestamp: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1974
|
-
);
|
|
1975
|
-
}
|
|
1976
|
-
await this.ensureAdminConnected();
|
|
1977
|
-
const byTopic = /* @__PURE__ */ new Map();
|
|
1978
|
-
for (const { topic: topic2, partition, timestamp } of assignments) {
|
|
1979
|
-
const list = byTopic.get(topic2) ?? [];
|
|
1980
|
-
list.push({ partition, timestamp });
|
|
1981
|
-
byTopic.set(topic2, list);
|
|
1982
|
-
}
|
|
1983
|
-
for (const [topic2, parts] of byTopic) {
|
|
1984
|
-
const offsets = await Promise.all(
|
|
1985
|
-
parts.map(async ({ partition, timestamp }) => {
|
|
1986
|
-
const results = await this.admin.fetchTopicOffsetsByTime(
|
|
1987
|
-
topic2,
|
|
1988
|
-
timestamp
|
|
1989
|
-
);
|
|
1990
|
-
const found = results.find(
|
|
1991
|
-
(r) => r.partition === partition
|
|
1992
|
-
);
|
|
1993
|
-
return { partition, offset: found?.offset ?? "-1" };
|
|
1994
|
-
})
|
|
1995
|
-
);
|
|
1996
|
-
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
|
|
1997
|
-
this.logger.log(
|
|
1998
|
-
`Offsets set by timestamp for group "${gid}" on "${topic2}": ${JSON.stringify(offsets)}`
|
|
1999
|
-
);
|
|
2000
|
-
}
|
|
2307
|
+
return this.adminOps.seekToTimestamp(groupId, assignments);
|
|
2001
2308
|
}
|
|
2002
2309
|
/**
|
|
2003
2310
|
* Returns the current circuit breaker state for a specific topic partition.
|
|
@@ -2011,14 +2318,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
2011
2318
|
* @returns `{ status, failures, windowSize }` snapshot for a given partition or `undefined` if no state exists.
|
|
2012
2319
|
*/
|
|
2013
2320
|
getCircuitState(topic2, partition, groupId) {
|
|
2014
|
-
|
|
2015
|
-
const state = this.circuitStates.get(`${gid}:${topic2}:${partition}`);
|
|
2016
|
-
if (!state) return void 0;
|
|
2017
|
-
return {
|
|
2018
|
-
status: state.status,
|
|
2019
|
-
failures: state.window.filter((v) => !v).length,
|
|
2020
|
-
windowSize: state.window.length
|
|
2021
|
-
};
|
|
2321
|
+
return this.circuitBreaker.getState(topic2, partition, groupId ?? this.defaultGroupId);
|
|
2022
2322
|
}
|
|
2023
2323
|
/**
|
|
2024
2324
|
* Query consumer group lag per partition.
|
|
@@ -2032,83 +2332,32 @@ var KafkaClient = class _KafkaClient {
|
|
|
2032
2332
|
* committed offset. Use `checkStatus()` to verify broker connectivity in that case.
|
|
2033
2333
|
*/
|
|
2034
2334
|
async getConsumerLag(groupId) {
|
|
2035
|
-
|
|
2036
|
-
await this.ensureAdminConnected();
|
|
2037
|
-
const committedByTopic = await this.admin.fetchOffsets({ groupId: gid });
|
|
2038
|
-
const brokerOffsetsAll = await Promise.all(
|
|
2039
|
-
committedByTopic.map(({ topic: topic2 }) => this.admin.fetchTopicOffsets(topic2))
|
|
2040
|
-
);
|
|
2041
|
-
const result = [];
|
|
2042
|
-
for (let i = 0; i < committedByTopic.length; i++) {
|
|
2043
|
-
const { topic: topic2, partitions } = committedByTopic[i];
|
|
2044
|
-
const brokerOffsets = brokerOffsetsAll[i];
|
|
2045
|
-
for (const { partition, offset } of partitions) {
|
|
2046
|
-
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
2047
|
-
if (!broker) continue;
|
|
2048
|
-
const committed = parseInt(offset, 10);
|
|
2049
|
-
const high = parseInt(broker.high, 10);
|
|
2050
|
-
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
2051
|
-
result.push({ topic: topic2, partition, lag });
|
|
2052
|
-
}
|
|
2053
|
-
}
|
|
2054
|
-
return result;
|
|
2335
|
+
return this.adminOps.getConsumerLag(groupId);
|
|
2055
2336
|
}
|
|
2056
2337
|
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
2057
2338
|
async checkStatus() {
|
|
2058
|
-
|
|
2059
|
-
await this.ensureAdminConnected();
|
|
2060
|
-
const topics = await this.admin.listTopics();
|
|
2061
|
-
return { status: "up", clientId: this.clientId, topics };
|
|
2062
|
-
} catch (error) {
|
|
2063
|
-
return {
|
|
2064
|
-
status: "down",
|
|
2065
|
-
clientId: this.clientId,
|
|
2066
|
-
error: error instanceof Error ? error.message : String(error)
|
|
2067
|
-
};
|
|
2068
|
-
}
|
|
2339
|
+
return this.adminOps.checkStatus();
|
|
2069
2340
|
}
|
|
2070
2341
|
/**
|
|
2071
2342
|
* List all consumer groups known to the broker.
|
|
2072
2343
|
* Useful for monitoring which groups are active and their current state.
|
|
2073
2344
|
*/
|
|
2074
2345
|
async listConsumerGroups() {
|
|
2075
|
-
|
|
2076
|
-
const result = await this.admin.listGroups();
|
|
2077
|
-
return result.groups.map((g) => ({
|
|
2078
|
-
groupId: g.groupId,
|
|
2079
|
-
state: g.state ?? "Unknown"
|
|
2080
|
-
}));
|
|
2346
|
+
return this.adminOps.listConsumerGroups();
|
|
2081
2347
|
}
|
|
2082
2348
|
/**
|
|
2083
2349
|
* Describe topics — returns partition layout, leader, replicas, and ISR.
|
|
2084
2350
|
* @param topics Topic names to describe. Omit to describe all topics.
|
|
2085
2351
|
*/
|
|
2086
2352
|
async describeTopics(topics) {
|
|
2087
|
-
|
|
2088
|
-
const result = await this.admin.fetchTopicMetadata(
|
|
2089
|
-
topics ? { topics } : void 0
|
|
2090
|
-
);
|
|
2091
|
-
return result.topics.map((t) => ({
|
|
2092
|
-
name: t.name,
|
|
2093
|
-
partitions: t.partitions.map((p) => ({
|
|
2094
|
-
partition: p.partitionId ?? p.partition,
|
|
2095
|
-
leader: p.leader,
|
|
2096
|
-
replicas: p.replicas.map(
|
|
2097
|
-
(r) => typeof r === "number" ? r : r.nodeId
|
|
2098
|
-
),
|
|
2099
|
-
isr: p.isr.map(
|
|
2100
|
-
(r) => typeof r === "number" ? r : r.nodeId
|
|
2101
|
-
)
|
|
2102
|
-
}))
|
|
2103
|
-
}));
|
|
2353
|
+
return this.adminOps.describeTopics(topics);
|
|
2104
2354
|
}
|
|
2105
2355
|
/**
|
|
2106
2356
|
* Delete records from a topic up to (but not including) the given offsets.
|
|
2107
2357
|
* All messages with offsets **before** the given offset are deleted.
|
|
2108
2358
|
*/
|
|
2109
2359
|
async deleteRecords(topic2, partitions) {
|
|
2110
|
-
|
|
2111
|
-
await this.admin.deleteTopicRecords({ topic: topic2, partitions });
|
|
2360
|
+
return this.adminOps.deleteRecords(topic2, partitions);
|
|
2112
2361
|
}
|
|
2113
2362
|
/** Return the client ID provided during `KafkaClient` construction. */
|
|
2114
2363
|
getClientId() {
|
|
@@ -2123,24 +2372,8 @@ var KafkaClient = class _KafkaClient {
|
|
|
2123
2372
|
* a zero-valued snapshot.
|
|
2124
2373
|
* @returns Read-only `KafkaMetrics` snapshot: `processedCount`, `retryCount`, `dlqCount`, `dedupCount`.
|
|
2125
2374
|
*/
|
|
2126
|
-
getMetrics(topic2) {
|
|
2127
|
-
|
|
2128
|
-
const m = this._topicMetrics.get(topic2);
|
|
2129
|
-
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
2130
|
-
}
|
|
2131
|
-
const agg = {
|
|
2132
|
-
processedCount: 0,
|
|
2133
|
-
retryCount: 0,
|
|
2134
|
-
dlqCount: 0,
|
|
2135
|
-
dedupCount: 0
|
|
2136
|
-
};
|
|
2137
|
-
for (const m of this._topicMetrics.values()) {
|
|
2138
|
-
agg.processedCount += m.processedCount;
|
|
2139
|
-
agg.retryCount += m.retryCount;
|
|
2140
|
-
agg.dlqCount += m.dlqCount;
|
|
2141
|
-
agg.dedupCount += m.dedupCount;
|
|
2142
|
-
}
|
|
2143
|
-
return agg;
|
|
2375
|
+
getMetrics(topic2) {
|
|
2376
|
+
return this.metrics.getMetrics(topic2);
|
|
2144
2377
|
}
|
|
2145
2378
|
/**
|
|
2146
2379
|
* Reset internal event counters to zero.
|
|
@@ -2148,15 +2381,11 @@ var KafkaClient = class _KafkaClient {
|
|
|
2148
2381
|
* @param topic Topic name to reset. When omitted, all topics are reset.
|
|
2149
2382
|
*/
|
|
2150
2383
|
resetMetrics(topic2) {
|
|
2151
|
-
|
|
2152
|
-
this._topicMetrics.delete(topic2);
|
|
2153
|
-
return;
|
|
2154
|
-
}
|
|
2155
|
-
this._topicMetrics.clear();
|
|
2384
|
+
this.metrics.resetMetrics(topic2);
|
|
2156
2385
|
}
|
|
2157
2386
|
/** Gracefully disconnect producer, all consumers, and admin. */
|
|
2158
2387
|
async disconnect(drainTimeoutMs = 3e4) {
|
|
2159
|
-
await this.waitForDrain(drainTimeoutMs);
|
|
2388
|
+
await this.inFlight.waitForDrain(drainTimeoutMs);
|
|
2160
2389
|
const tasks = [this.producer.disconnect()];
|
|
2161
2390
|
if (this.txProducer) {
|
|
2162
2391
|
tasks.push(this.txProducer.disconnect());
|
|
@@ -2164,28 +2393,17 @@ var KafkaClient = class _KafkaClient {
|
|
|
2164
2393
|
this.txProducer = void 0;
|
|
2165
2394
|
this.txProducerInitPromise = void 0;
|
|
2166
2395
|
}
|
|
2167
|
-
for (const txId of this.retryTxProducers.keys())
|
|
2168
|
-
|
|
2169
|
-
}
|
|
2170
|
-
for (const p of this.retryTxProducers.values()) {
|
|
2171
|
-
tasks.push(p.disconnect());
|
|
2172
|
-
}
|
|
2396
|
+
for (const txId of this.retryTxProducers.keys()) _activeTransactionalIds.delete(txId);
|
|
2397
|
+
for (const p of this.retryTxProducers.values()) tasks.push(p.disconnect());
|
|
2173
2398
|
this.retryTxProducers.clear();
|
|
2174
|
-
for (const consumer of this.consumers.values())
|
|
2175
|
-
|
|
2176
|
-
}
|
|
2177
|
-
if (this.isAdminConnected) {
|
|
2178
|
-
tasks.push(this.admin.disconnect());
|
|
2179
|
-
this.isAdminConnected = false;
|
|
2180
|
-
}
|
|
2399
|
+
for (const consumer of this.consumers.values()) tasks.push(consumer.disconnect());
|
|
2400
|
+
tasks.push(this.adminOps.disconnect());
|
|
2181
2401
|
await Promise.allSettled(tasks);
|
|
2182
2402
|
this.consumers.clear();
|
|
2183
2403
|
this.runningConsumers.clear();
|
|
2184
2404
|
this.consumerCreationOptions.clear();
|
|
2185
2405
|
this.companionGroupIds.clear();
|
|
2186
|
-
|
|
2187
|
-
this.circuitStates.clear();
|
|
2188
|
-
this.circuitConfigs.clear();
|
|
2406
|
+
this.circuitBreaker.clear();
|
|
2189
2407
|
this.logger.log("All connections closed");
|
|
2190
2408
|
}
|
|
2191
2409
|
// ── Graceful shutdown ────────────────────────────────────────────
|
|
@@ -2204,235 +2422,20 @@ var KafkaClient = class _KafkaClient {
|
|
|
2204
2422
|
*/
|
|
2205
2423
|
enableGracefulShutdown(signals = ["SIGTERM", "SIGINT"], drainTimeoutMs = 3e4) {
|
|
2206
2424
|
const handler = () => {
|
|
2207
|
-
this.logger.log(
|
|
2208
|
-
"Shutdown signal received \u2014 draining in-flight handlers..."
|
|
2209
|
-
);
|
|
2425
|
+
this.logger.log("Shutdown signal received \u2014 draining in-flight handlers...");
|
|
2210
2426
|
this.disconnect(drainTimeoutMs).catch(
|
|
2211
|
-
(err) => this.logger.error(
|
|
2212
|
-
"Error during graceful shutdown:",
|
|
2213
|
-
toError(err).message
|
|
2214
|
-
)
|
|
2427
|
+
(err) => this.logger.error("Error during graceful shutdown:", toError(err).message)
|
|
2215
2428
|
);
|
|
2216
2429
|
};
|
|
2217
|
-
for (const signal of signals)
|
|
2218
|
-
process.once(signal, handler);
|
|
2219
|
-
}
|
|
2220
|
-
}
|
|
2221
|
-
/**
|
|
2222
|
-
* Increment the in-flight handler count and return a promise that calls the given handler.
|
|
2223
|
-
* When the promise resolves or rejects, decrement the in flight handler count.
|
|
2224
|
-
* If the in flight handler count reaches 0, call all previously registered drain resolvers.
|
|
2225
|
-
* @param fn The handler to call when the promise is resolved or rejected.
|
|
2226
|
-
* @returns A promise that resolves or rejects with the result of calling the handler.
|
|
2227
|
-
*/
|
|
2228
|
-
trackInFlight(fn) {
|
|
2229
|
-
this.inFlightTotal++;
|
|
2230
|
-
return fn().finally(() => {
|
|
2231
|
-
this.inFlightTotal--;
|
|
2232
|
-
if (this.inFlightTotal === 0) {
|
|
2233
|
-
this.drainResolvers.splice(0).forEach((r) => r());
|
|
2234
|
-
}
|
|
2235
|
-
});
|
|
2236
|
-
}
|
|
2237
|
-
/**
|
|
2238
|
-
* Waits for all in-flight handlers to complete or for a given timeout, whichever comes first.
|
|
2239
|
-
* @param timeoutMs Maximum time to wait in milliseconds.
|
|
2240
|
-
* @returns A promise that resolves when all handlers have completed or the timeout is reached.
|
|
2241
|
-
* @private
|
|
2242
|
-
*/
|
|
2243
|
-
waitForDrain(timeoutMs) {
|
|
2244
|
-
if (this.inFlightTotal === 0) return Promise.resolve();
|
|
2245
|
-
return new Promise((resolve) => {
|
|
2246
|
-
let handle;
|
|
2247
|
-
const onDrain = () => {
|
|
2248
|
-
clearTimeout(handle);
|
|
2249
|
-
resolve();
|
|
2250
|
-
};
|
|
2251
|
-
this.drainResolvers.push(onDrain);
|
|
2252
|
-
handle = setTimeout(() => {
|
|
2253
|
-
const idx = this.drainResolvers.indexOf(onDrain);
|
|
2254
|
-
if (idx !== -1) this.drainResolvers.splice(idx, 1);
|
|
2255
|
-
this.logger.warn(
|
|
2256
|
-
`Drain timed out after ${timeoutMs}ms \u2014 ${this.inFlightTotal} handler(s) still in flight`
|
|
2257
|
-
);
|
|
2258
|
-
resolve();
|
|
2259
|
-
}, timeoutMs);
|
|
2260
|
-
});
|
|
2430
|
+
for (const signal of signals) process.once(signal, handler);
|
|
2261
2431
|
}
|
|
2262
2432
|
// ── Private helpers ──────────────────────────────────────────────
|
|
2263
|
-
/**
|
|
2264
|
-
* Prepare a send payload by registering the topic's schema and then building the payload.
|
|
2265
|
-
* @param topicOrDesc - topic name or topic descriptor
|
|
2266
|
-
* @param messages - batch of messages to send
|
|
2267
|
-
* @returns - prepared payload
|
|
2268
|
-
*/
|
|
2269
2433
|
async preparePayload(topicOrDesc, messages, compression) {
|
|
2270
2434
|
registerSchema(topicOrDesc, this.schemaRegistry, this.logger);
|
|
2271
|
-
const payload = await buildSendPayload(
|
|
2272
|
-
topicOrDesc,
|
|
2273
|
-
messages,
|
|
2274
|
-
this.producerOpsDeps,
|
|
2275
|
-
compression
|
|
2276
|
-
);
|
|
2435
|
+
const payload = await buildSendPayload(topicOrDesc, messages, this._producerOpsDeps, compression);
|
|
2277
2436
|
await this.ensureTopic(payload.topic);
|
|
2278
2437
|
return payload;
|
|
2279
2438
|
}
|
|
2280
|
-
// afterSend is called once per message — symmetric with beforeSend in buildSendPayload.
|
|
2281
|
-
notifyAfterSend(topic2, count) {
|
|
2282
|
-
for (let i = 0; i < count; i++) {
|
|
2283
|
-
for (const inst of this.instrumentation) {
|
|
2284
|
-
inst.afterSend?.(topic2);
|
|
2285
|
-
}
|
|
2286
|
-
}
|
|
2287
|
-
}
|
|
2288
|
-
/**
|
|
2289
|
-
* Returns the KafkaMetrics for a given topic.
|
|
2290
|
-
* If the topic hasn't seen any events, initializes a zero-valued snapshot.
|
|
2291
|
-
* @param topic - name of the topic to get the metrics for
|
|
2292
|
-
* @returns - KafkaMetrics for the given topic
|
|
2293
|
-
*/
|
|
2294
|
-
metricsFor(topic2) {
|
|
2295
|
-
let m = this._topicMetrics.get(topic2);
|
|
2296
|
-
if (!m) {
|
|
2297
|
-
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
2298
|
-
this._topicMetrics.set(topic2, m);
|
|
2299
|
-
}
|
|
2300
|
-
return m;
|
|
2301
|
-
}
|
|
2302
|
-
/**
|
|
2303
|
-
* Notifies instrumentation hooks of a retry event.
|
|
2304
|
-
* @param envelope The original message envelope that triggered the retry.
|
|
2305
|
-
* @param attempt The current retry attempt (1-indexed).
|
|
2306
|
-
* @param maxRetries The maximum number of retries configured for this topic.
|
|
2307
|
-
*/
|
|
2308
|
-
notifyRetry(envelope, attempt, maxRetries) {
|
|
2309
|
-
this.metricsFor(envelope.topic).retryCount++;
|
|
2310
|
-
for (const inst of this.instrumentation) {
|
|
2311
|
-
inst.onRetry?.(envelope, attempt, maxRetries);
|
|
2312
|
-
}
|
|
2313
|
-
}
|
|
2314
|
-
/**
|
|
2315
|
-
* Called whenever a message is routed to the dead letter queue.
|
|
2316
|
-
* @param envelope The original message envelope.
|
|
2317
|
-
* @param reason The reason for routing to the dead letter queue.
|
|
2318
|
-
* @param gid The group ID of the consumer that triggered the circuit breaker, if any.
|
|
2319
|
-
*/
|
|
2320
|
-
notifyDlq(envelope, reason, gid) {
|
|
2321
|
-
this.metricsFor(envelope.topic).dlqCount++;
|
|
2322
|
-
for (const inst of this.instrumentation) {
|
|
2323
|
-
inst.onDlq?.(envelope, reason);
|
|
2324
|
-
}
|
|
2325
|
-
if (!gid) return;
|
|
2326
|
-
const cfg = this.circuitConfigs.get(gid);
|
|
2327
|
-
if (!cfg) return;
|
|
2328
|
-
const threshold = cfg.threshold ?? 5;
|
|
2329
|
-
const recoveryMs = cfg.recoveryMs ?? 3e4;
|
|
2330
|
-
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
2331
|
-
let state = this.circuitStates.get(stateKey);
|
|
2332
|
-
if (!state) {
|
|
2333
|
-
state = { status: "closed", window: [], successes: 0 };
|
|
2334
|
-
this.circuitStates.set(stateKey, state);
|
|
2335
|
-
}
|
|
2336
|
-
if (state.status === "open") return;
|
|
2337
|
-
const openCircuit = () => {
|
|
2338
|
-
state.status = "open";
|
|
2339
|
-
state.window = [];
|
|
2340
|
-
state.successes = 0;
|
|
2341
|
-
clearTimeout(state.timer);
|
|
2342
|
-
for (const inst of this.instrumentation)
|
|
2343
|
-
inst.onCircuitOpen?.(envelope.topic, envelope.partition);
|
|
2344
|
-
this.pauseConsumer(gid, [
|
|
2345
|
-
{ topic: envelope.topic, partitions: [envelope.partition] }
|
|
2346
|
-
]);
|
|
2347
|
-
state.timer = setTimeout(() => {
|
|
2348
|
-
state.status = "half-open";
|
|
2349
|
-
state.successes = 0;
|
|
2350
|
-
this.logger.log(
|
|
2351
|
-
`[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2352
|
-
);
|
|
2353
|
-
for (const inst of this.instrumentation)
|
|
2354
|
-
inst.onCircuitHalfOpen?.(envelope.topic, envelope.partition);
|
|
2355
|
-
this.resumeConsumer(gid, [
|
|
2356
|
-
{ topic: envelope.topic, partitions: [envelope.partition] }
|
|
2357
|
-
]);
|
|
2358
|
-
}, recoveryMs);
|
|
2359
|
-
};
|
|
2360
|
-
if (state.status === "half-open") {
|
|
2361
|
-
clearTimeout(state.timer);
|
|
2362
|
-
this.logger.warn(
|
|
2363
|
-
`[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2364
|
-
);
|
|
2365
|
-
openCircuit();
|
|
2366
|
-
return;
|
|
2367
|
-
}
|
|
2368
|
-
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
2369
|
-
state.window = [...state.window, false];
|
|
2370
|
-
if (state.window.length > windowSize) {
|
|
2371
|
-
state.window = state.window.slice(state.window.length - windowSize);
|
|
2372
|
-
}
|
|
2373
|
-
const failures = state.window.filter((v) => !v).length;
|
|
2374
|
-
if (failures >= threshold) {
|
|
2375
|
-
this.logger.warn(
|
|
2376
|
-
`[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
|
|
2377
|
-
);
|
|
2378
|
-
openCircuit();
|
|
2379
|
-
}
|
|
2380
|
-
}
|
|
2381
|
-
/**
|
|
2382
|
-
* Notify all instrumentation hooks about a duplicate message detection.
|
|
2383
|
-
* Invoked by the consumer after a message has been successfully processed
|
|
2384
|
-
* and the Lamport clock detected a duplicate.
|
|
2385
|
-
* @param envelope The processed message envelope.
|
|
2386
|
-
* @param strategy The duplicate detection strategy used.
|
|
2387
|
-
*/
|
|
2388
|
-
notifyDuplicate(envelope, strategy) {
|
|
2389
|
-
this.metricsFor(envelope.topic).dedupCount++;
|
|
2390
|
-
for (const inst of this.instrumentation) {
|
|
2391
|
-
inst.onDuplicate?.(envelope, strategy);
|
|
2392
|
-
}
|
|
2393
|
-
}
|
|
2394
|
-
/**
|
|
2395
|
-
* Notify all instrumentation hooks about a successfully processed message.
|
|
2396
|
-
* Invoked by the consumer after a message has been successfully processed
|
|
2397
|
-
* by the handler.
|
|
2398
|
-
* @param envelope The processed message envelope.
|
|
2399
|
-
* @param gid The optional consumer group ID.
|
|
2400
|
-
*/
|
|
2401
|
-
notifyMessage(envelope, gid) {
|
|
2402
|
-
this.metricsFor(envelope.topic).processedCount++;
|
|
2403
|
-
for (const inst of this.instrumentation) {
|
|
2404
|
-
inst.onMessage?.(envelope);
|
|
2405
|
-
}
|
|
2406
|
-
if (!gid) return;
|
|
2407
|
-
const cfg = this.circuitConfigs.get(gid);
|
|
2408
|
-
if (!cfg) return;
|
|
2409
|
-
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
2410
|
-
const state = this.circuitStates.get(stateKey);
|
|
2411
|
-
if (!state) return;
|
|
2412
|
-
const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
|
|
2413
|
-
if (state.status === "half-open") {
|
|
2414
|
-
state.successes++;
|
|
2415
|
-
if (state.successes >= halfOpenSuccesses) {
|
|
2416
|
-
clearTimeout(state.timer);
|
|
2417
|
-
state.timer = void 0;
|
|
2418
|
-
state.status = "closed";
|
|
2419
|
-
state.window = [];
|
|
2420
|
-
state.successes = 0;
|
|
2421
|
-
this.logger.log(
|
|
2422
|
-
`[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2423
|
-
);
|
|
2424
|
-
for (const inst of this.instrumentation)
|
|
2425
|
-
inst.onCircuitClose?.(envelope.topic, envelope.partition);
|
|
2426
|
-
}
|
|
2427
|
-
} else if (state.status === "closed") {
|
|
2428
|
-
const threshold = cfg.threshold ?? 5;
|
|
2429
|
-
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
2430
|
-
state.window = [...state.window, true];
|
|
2431
|
-
if (state.window.length > windowSize) {
|
|
2432
|
-
state.window = state.window.slice(state.window.length - windowSize);
|
|
2433
|
-
}
|
|
2434
|
-
}
|
|
2435
|
-
}
|
|
2436
2439
|
/**
|
|
2437
2440
|
* Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
|
|
2438
2441
|
* The handler itself is not cancelled — the warning is diagnostic only.
|
|
@@ -2443,79 +2446,10 @@ var KafkaClient = class _KafkaClient {
|
|
|
2443
2446
|
if (timer !== void 0) clearTimeout(timer);
|
|
2444
2447
|
});
|
|
2445
2448
|
timer = setTimeout(() => {
|
|
2446
|
-
this.logger.warn(
|
|
2447
|
-
`Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
|
|
2448
|
-
);
|
|
2449
|
+
this.logger.warn(`Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`);
|
|
2449
2450
|
}, timeoutMs);
|
|
2450
2451
|
return promise;
|
|
2451
2452
|
}
|
|
2452
|
-
/**
|
|
2453
|
-
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
2454
|
-
* `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
|
|
2455
|
-
* rather than silently discovering missing topics on the first handler failure.
|
|
2456
|
-
*/
|
|
2457
|
-
async validateRetryTopicsExist(topicNames, maxRetries) {
|
|
2458
|
-
await this.ensureAdminConnected();
|
|
2459
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2460
|
-
const missing = [];
|
|
2461
|
-
for (const t of topicNames) {
|
|
2462
|
-
for (let level = 1; level <= maxRetries; level++) {
|
|
2463
|
-
const retryTopic = `${t}.retry.${level}`;
|
|
2464
|
-
if (!existing.has(retryTopic)) missing.push(retryTopic);
|
|
2465
|
-
}
|
|
2466
|
-
}
|
|
2467
|
-
if (missing.length > 0) {
|
|
2468
|
-
throw new Error(
|
|
2469
|
-
`retryTopics: true but the following retry topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2470
|
-
);
|
|
2471
|
-
}
|
|
2472
|
-
}
|
|
2473
|
-
/**
|
|
2474
|
-
* When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
|
|
2475
|
-
* consumed topic. Throws a clear error at startup rather than silently discovering
|
|
2476
|
-
* missing DLQ topics on the first handler failure.
|
|
2477
|
-
*/
|
|
2478
|
-
async validateDlqTopicsExist(topicNames) {
|
|
2479
|
-
await this.ensureAdminConnected();
|
|
2480
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2481
|
-
const missing = topicNames.filter((t) => !existing.has(`${t}.dlq`)).map((t) => `${t}.dlq`);
|
|
2482
|
-
if (missing.length > 0) {
|
|
2483
|
-
throw new Error(
|
|
2484
|
-
`dlq: true but the following DLQ topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2485
|
-
);
|
|
2486
|
-
}
|
|
2487
|
-
}
|
|
2488
|
-
/**
|
|
2489
|
-
* When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
|
|
2490
|
-
* that every `<topic>.duplicates` destination topic already exists. Throws a
|
|
2491
|
-
* clear error at startup rather than silently dropping duplicates on first hit.
|
|
2492
|
-
*/
|
|
2493
|
-
async validateDuplicatesTopicsExist(topicNames, customDestination) {
|
|
2494
|
-
await this.ensureAdminConnected();
|
|
2495
|
-
const existing = new Set(await this.admin.listTopics());
|
|
2496
|
-
const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
|
|
2497
|
-
const missing = toCheck.filter((t) => !existing.has(t));
|
|
2498
|
-
if (missing.length > 0) {
|
|
2499
|
-
throw new Error(
|
|
2500
|
-
`deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
2501
|
-
);
|
|
2502
|
-
}
|
|
2503
|
-
}
|
|
2504
|
-
/**
|
|
2505
|
-
* Connect the admin client if not already connected.
|
|
2506
|
-
* The flag is only set to `true` after a successful connect — if `admin.connect()`
|
|
2507
|
-
* throws the flag remains `false` so the next call will retry the connection.
|
|
2508
|
-
*/
|
|
2509
|
-
async ensureAdminConnected() {
|
|
2510
|
-
if (this.isAdminConnected) return;
|
|
2511
|
-
try {
|
|
2512
|
-
await this.admin.connect();
|
|
2513
|
-
this.isAdminConnected = true;
|
|
2514
|
-
} catch (err) {
|
|
2515
|
-
this.isAdminConnected = false;
|
|
2516
|
-
throw err;
|
|
2517
|
-
}
|
|
2518
|
-
}
|
|
2519
2453
|
/**
|
|
2520
2454
|
* Create and connect a transactional producer for EOS retry routing.
|
|
2521
2455
|
* Each retry level consumer gets its own producer with a unique `transactionalId`
|
|
@@ -2528,12 +2462,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
2528
2462
|
);
|
|
2529
2463
|
}
|
|
2530
2464
|
const p = this.kafka.producer({
|
|
2531
|
-
kafkaJS: {
|
|
2532
|
-
acks: -1,
|
|
2533
|
-
idempotent: true,
|
|
2534
|
-
transactionalId,
|
|
2535
|
-
maxInFlightRequests: 1
|
|
2536
|
-
}
|
|
2465
|
+
kafkaJS: { acks: -1, idempotent: true, transactionalId, maxInFlightRequests: 1 }
|
|
2537
2466
|
});
|
|
2538
2467
|
await p.connect();
|
|
2539
2468
|
_activeTransactionalIds.add(transactionalId);
|
|
@@ -2542,20 +2471,16 @@ var KafkaClient = class _KafkaClient {
|
|
|
2542
2471
|
}
|
|
2543
2472
|
/**
|
|
2544
2473
|
* Ensure that a topic exists by creating it if it doesn't already exist.
|
|
2545
|
-
* If `autoCreateTopics` is disabled,
|
|
2546
|
-
*
|
|
2547
|
-
* If multiple concurrent calls are made to `ensureTopic` for the same topic,
|
|
2548
|
-
* they are deduplicated to prevent multiple calls to `admin.createTopics()`.
|
|
2549
|
-
* @param topic - The topic to ensure exists.
|
|
2550
|
-
* @returns A promise that resolves when the topic has been created or already exists.
|
|
2474
|
+
* If `autoCreateTopics` is disabled, returns immediately.
|
|
2475
|
+
* Concurrent calls for the same topic are deduplicated.
|
|
2551
2476
|
*/
|
|
2552
2477
|
async ensureTopic(topic2) {
|
|
2553
2478
|
if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
|
|
2554
2479
|
let p = this.ensureTopicPromises.get(topic2);
|
|
2555
2480
|
if (!p) {
|
|
2556
2481
|
p = (async () => {
|
|
2557
|
-
await this.
|
|
2558
|
-
await this.admin.createTopics({
|
|
2482
|
+
await this.adminOps.ensureConnected();
|
|
2483
|
+
await this.adminOps.admin.createTopics({
|
|
2559
2484
|
topics: [{ topic: topic2, numPartitions: this.numPartitions }]
|
|
2560
2485
|
});
|
|
2561
2486
|
this.ensuredTopics.add(topic2);
|
|
@@ -2595,99 +2520,78 @@ var KafkaClient = class _KafkaClient {
|
|
|
2595
2520
|
gid,
|
|
2596
2521
|
fromBeginning,
|
|
2597
2522
|
options.autoCommit ?? true,
|
|
2598
|
-
this.
|
|
2523
|
+
this._consumerOpsDeps,
|
|
2599
2524
|
options.partitionAssigner
|
|
2600
2525
|
);
|
|
2601
|
-
const schemaMap = buildSchemaMap(
|
|
2602
|
-
stringTopics,
|
|
2603
|
-
this.schemaRegistry,
|
|
2604
|
-
optionSchemas,
|
|
2605
|
-
this.logger
|
|
2606
|
-
);
|
|
2526
|
+
const schemaMap = buildSchemaMap(stringTopics, this.schemaRegistry, optionSchemas, this.logger);
|
|
2607
2527
|
const topicNames = stringTopics.map((t) => resolveTopicName(t));
|
|
2608
|
-
const subscribeTopics = [
|
|
2609
|
-
|
|
2610
|
-
...regexTopics
|
|
2611
|
-
];
|
|
2612
|
-
for (const t of topicNames) {
|
|
2613
|
-
await this.ensureTopic(t);
|
|
2614
|
-
}
|
|
2528
|
+
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2529
|
+
for (const t of topicNames) await this.ensureTopic(t);
|
|
2615
2530
|
if (dlq) {
|
|
2616
|
-
for (const t of topicNames) {
|
|
2617
|
-
await this.ensureTopic(`${t}.dlq`);
|
|
2618
|
-
}
|
|
2531
|
+
for (const t of topicNames) await this.ensureTopic(`${t}.dlq`);
|
|
2619
2532
|
if (!this.autoCreateTopicsEnabled && topicNames.length > 0) {
|
|
2620
|
-
await this.validateDlqTopicsExist(topicNames);
|
|
2533
|
+
await this.adminOps.validateDlqTopicsExist(topicNames);
|
|
2621
2534
|
}
|
|
2622
2535
|
}
|
|
2623
2536
|
if (options.deduplication?.strategy === "topic") {
|
|
2624
2537
|
const dest = options.deduplication.duplicatesTopic;
|
|
2625
2538
|
if (this.autoCreateTopicsEnabled) {
|
|
2626
|
-
for (const t of topicNames) {
|
|
2627
|
-
await this.ensureTopic(dest ?? `${t}.duplicates`);
|
|
2628
|
-
}
|
|
2539
|
+
for (const t of topicNames) await this.ensureTopic(dest ?? `${t}.duplicates`);
|
|
2629
2540
|
} else if (topicNames.length > 0) {
|
|
2630
|
-
await this.validateDuplicatesTopicsExist(topicNames, dest);
|
|
2541
|
+
await this.adminOps.validateDuplicatesTopicsExist(topicNames, dest);
|
|
2631
2542
|
}
|
|
2632
2543
|
}
|
|
2633
2544
|
await consumer.connect();
|
|
2634
|
-
await subscribeWithRetry(
|
|
2635
|
-
consumer,
|
|
2636
|
-
subscribeTopics,
|
|
2637
|
-
this.logger,
|
|
2638
|
-
options.subscribeRetry
|
|
2639
|
-
);
|
|
2545
|
+
await subscribeWithRetry(consumer, subscribeTopics, this.logger, options.subscribeRetry);
|
|
2640
2546
|
const displayTopics = subscribeTopics.map((t) => t instanceof RegExp ? t.toString() : t).join(", ");
|
|
2641
|
-
this.logger.log(
|
|
2642
|
-
`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
|
|
2643
|
-
);
|
|
2547
|
+
this.logger.log(`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`);
|
|
2644
2548
|
return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex };
|
|
2645
2549
|
}
|
|
2646
2550
|
/** Create or retrieve the deduplication context for a consumer group. */
|
|
2647
2551
|
resolveDeduplicationContext(groupId, options) {
|
|
2648
2552
|
if (!options) return void 0;
|
|
2649
|
-
if (!this.dedupStates.has(groupId))
|
|
2650
|
-
this.dedupStates.set(groupId, /* @__PURE__ */ new Map());
|
|
2651
|
-
}
|
|
2553
|
+
if (!this.dedupStates.has(groupId)) this.dedupStates.set(groupId, /* @__PURE__ */ new Map());
|
|
2652
2554
|
return { options, state: this.dedupStates.get(groupId) };
|
|
2653
2555
|
}
|
|
2654
|
-
// ──
|
|
2655
|
-
/**
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
strictSchemasEnabled: this.strictSchemasEnabled,
|
|
2668
|
-
instrumentation: this.instrumentation,
|
|
2669
|
-
logger: this.logger,
|
|
2670
|
-
nextLamportClock: () => ++this._lamportClock
|
|
2671
|
-
};
|
|
2556
|
+
// ── Shared consumer setup helpers ────────────────────────────────
|
|
2557
|
+
/** Guard checks shared by startConsumer and startBatchConsumer. */
|
|
2558
|
+
validateTopicConsumerOpts(topics, options) {
|
|
2559
|
+
if (options.retryTopics && !options.retry) {
|
|
2560
|
+
throw new Error(
|
|
2561
|
+
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
2562
|
+
);
|
|
2563
|
+
}
|
|
2564
|
+
if (options.retryTopics && topics.some((t) => t instanceof RegExp)) {
|
|
2565
|
+
throw new Error(
|
|
2566
|
+
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
2567
|
+
);
|
|
2568
|
+
}
|
|
2672
2569
|
}
|
|
2673
|
-
/**
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2570
|
+
/** Create EOS transactional producer context for atomic main → retry.1 routing. */
|
|
2571
|
+
async makeEosMainContext(gid, consumer, options) {
|
|
2572
|
+
if (!options.retryTopics || !options.retry) return void 0;
|
|
2573
|
+
const txProducer = await this.createRetryTxProducer(`${gid}-main-tx`);
|
|
2574
|
+
return { txProducer, consumer };
|
|
2575
|
+
}
|
|
2576
|
+
/** Start companion retry-level consumers and register them under the main groupId. */
|
|
2577
|
+
async launchRetryChain(gid, topicNames, handleMessage, retry, dlq, interceptors, schemaMap, assignmentTimeoutMs) {
|
|
2578
|
+
if (!this.autoCreateTopicsEnabled) {
|
|
2579
|
+
await this.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
2580
|
+
}
|
|
2581
|
+
const companions = await startRetryTopicConsumers(
|
|
2582
|
+
topicNames,
|
|
2583
|
+
gid,
|
|
2584
|
+
handleMessage,
|
|
2585
|
+
retry,
|
|
2586
|
+
dlq,
|
|
2587
|
+
interceptors,
|
|
2588
|
+
schemaMap,
|
|
2589
|
+
this._retryTopicDeps,
|
|
2590
|
+
assignmentTimeoutMs
|
|
2591
|
+
);
|
|
2592
|
+
this.companionGroupIds.set(gid, companions);
|
|
2690
2593
|
}
|
|
2594
|
+
// ── Deps object builders ─────────────────────────────────────────
|
|
2691
2595
|
/** Build MessageHandlerDeps with circuit breaker callbacks bound to the given groupId. */
|
|
2692
2596
|
messageDepsFor(gid) {
|
|
2693
2597
|
return {
|
|
@@ -2696,38 +2600,24 @@ var KafkaClient = class _KafkaClient {
|
|
|
2696
2600
|
instrumentation: this.instrumentation,
|
|
2697
2601
|
onMessageLost: this.onMessageLost,
|
|
2698
2602
|
onTtlExpired: this.onTtlExpired,
|
|
2699
|
-
onRetry: this.notifyRetry.bind(this),
|
|
2700
|
-
onDlq: (envelope, reason) => this.notifyDlq(envelope, reason, gid),
|
|
2701
|
-
onDuplicate: this.notifyDuplicate.bind(this),
|
|
2702
|
-
onMessage: (envelope) => this.notifyMessage(envelope, gid)
|
|
2603
|
+
onRetry: this.metrics.notifyRetry.bind(this.metrics),
|
|
2604
|
+
onDlq: (envelope, reason) => this.metrics.notifyDlq(envelope, reason, gid),
|
|
2605
|
+
onDuplicate: this.metrics.notifyDuplicate.bind(this.metrics),
|
|
2606
|
+
onMessage: (envelope) => this.metrics.notifyMessage(envelope, gid)
|
|
2703
2607
|
};
|
|
2704
2608
|
}
|
|
2705
|
-
/**
|
|
2706
|
-
|
|
2707
|
-
*
|
|
2708
|
-
* `logger`: The logger instance passed to the retry topic consumers.
|
|
2709
|
-
* `producer`: The producer instance passed to the retry topic consumers.
|
|
2710
|
-
* `instrumentation`: The instrumentation instance passed to the retry topic consumers.
|
|
2711
|
-
* `onMessageLost`: The callback function passed to the retry topic consumers for tracking lost messages.
|
|
2712
|
-
* `onRetry`: The callback function passed to the retry topic consumers for tracking retry attempts.
|
|
2713
|
-
* `onDlq`: The callback function passed to the retry topic consumers for tracking dead-letter queue routing.
|
|
2714
|
-
* `onMessage`: The callback function passed to the retry topic consumers for tracking message delivery.
|
|
2715
|
-
* `ensureTopic`: A function that ensures a topic exists before subscribing to it.
|
|
2716
|
-
* `getOrCreateConsumer`: A function that creates or retrieves a consumer instance.
|
|
2717
|
-
* `runningConsumers`: A map of consumer group IDs to their corresponding consumer instances.
|
|
2718
|
-
* `createRetryTxProducer`: A function that creates a retry transactional producer instance.
|
|
2719
|
-
*/
|
|
2720
|
-
get retryTopicDeps() {
|
|
2609
|
+
/** Build the deps object passed to retry topic consumers. */
|
|
2610
|
+
buildRetryTopicDeps() {
|
|
2721
2611
|
return {
|
|
2722
2612
|
logger: this.logger,
|
|
2723
2613
|
producer: this.producer,
|
|
2724
2614
|
instrumentation: this.instrumentation,
|
|
2725
2615
|
onMessageLost: this.onMessageLost,
|
|
2726
|
-
onRetry: this.notifyRetry.bind(this),
|
|
2727
|
-
onDlq: this.notifyDlq.bind(this),
|
|
2728
|
-
onMessage: this.notifyMessage.bind(this),
|
|
2616
|
+
onRetry: this.metrics.notifyRetry.bind(this.metrics),
|
|
2617
|
+
onDlq: this.metrics.notifyDlq.bind(this.metrics),
|
|
2618
|
+
onMessage: this.metrics.notifyMessage.bind(this.metrics),
|
|
2729
2619
|
ensureTopic: (t) => this.ensureTopic(t),
|
|
2730
|
-
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.
|
|
2620
|
+
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this._consumerOpsDeps),
|
|
2731
2621
|
runningConsumers: this.runningConsumers,
|
|
2732
2622
|
createRetryTxProducer: (txId) => this.createRetryTxProducer(txId)
|
|
2733
2623
|
};
|
|
@@ -2768,4 +2658,4 @@ export {
|
|
|
2768
2658
|
KafkaClient,
|
|
2769
2659
|
topic
|
|
2770
2660
|
};
|
|
2771
|
-
//# sourceMappingURL=chunk-
|
|
2661
|
+
//# sourceMappingURL=chunk-XP7LLRGQ.mjs.map
|