@drarzter/kafka-client 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -17
- package/dist/chunk-TD2AE774.mjs +1231 -0
- package/dist/chunk-TD2AE774.mjs.map +1 -0
- package/dist/core.d.mts +14 -41
- package/dist/core.d.ts +14 -41
- package/dist/core.js +652 -511
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +1 -1
- package/dist/index.d.mts +17 -23
- package/dist/index.d.ts +17 -23
- package/dist/index.js +692 -569
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +41 -59
- package/dist/index.mjs.map +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/otel.d.ts +1 -1
- package/dist/otel.js +9 -4
- package/dist/otel.js.map +1 -1
- package/dist/otel.mjs +9 -4
- package/dist/otel.mjs.map +1 -1
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +6 -6
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +6 -6
- package/dist/testing.mjs.map +1 -1
- package/dist/{envelope-BpyKN_WL.d.mts → types-DwERZ6ql.d.mts} +99 -83
- package/dist/{envelope-BpyKN_WL.d.ts → types-DwERZ6ql.d.ts} +99 -83
- package/package.json +1 -1
- package/dist/chunk-Z3O5GTS7.mjs +0 -1090
- package/dist/chunk-Z3O5GTS7.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -55,10 +55,10 @@ __export(index_exports, {
|
|
|
55
55
|
});
|
|
56
56
|
module.exports = __toCommonJS(index_exports);
|
|
57
57
|
|
|
58
|
-
// src/client/kafka.client.ts
|
|
58
|
+
// src/client/kafka.client/index.ts
|
|
59
59
|
var import_kafka_javascript = require("@confluentinc/kafka-javascript");
|
|
60
60
|
|
|
61
|
-
// src/client/envelope.ts
|
|
61
|
+
// src/client/message/envelope.ts
|
|
62
62
|
var import_node_async_hooks = require("async_hooks");
|
|
63
63
|
var import_node_crypto = require("crypto");
|
|
64
64
|
var HEADER_EVENT_ID = "x-event-id";
|
|
@@ -150,7 +150,123 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
|
150
150
|
}
|
|
151
151
|
};
|
|
152
152
|
|
|
153
|
-
// src/client/
|
|
153
|
+
// src/client/kafka.client/producer-ops.ts
|
|
154
|
+
function resolveTopicName(topicOrDescriptor) {
|
|
155
|
+
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
156
|
+
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
157
|
+
return topicOrDescriptor.__topic;
|
|
158
|
+
}
|
|
159
|
+
return String(topicOrDescriptor);
|
|
160
|
+
}
|
|
161
|
+
function registerSchema(topicOrDesc, schemaRegistry) {
|
|
162
|
+
if (topicOrDesc?.__schema) {
|
|
163
|
+
const topic2 = resolveTopicName(topicOrDesc);
|
|
164
|
+
schemaRegistry.set(topic2, topicOrDesc.__schema);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
async function validateMessage(topicOrDesc, message, deps) {
|
|
168
|
+
const topicName = resolveTopicName(topicOrDesc);
|
|
169
|
+
if (topicOrDesc?.__schema) {
|
|
170
|
+
try {
|
|
171
|
+
return await topicOrDesc.__schema.parse(message);
|
|
172
|
+
} catch (error) {
|
|
173
|
+
throw new KafkaValidationError(topicName, message, {
|
|
174
|
+
cause: error instanceof Error ? error : new Error(String(error))
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (deps.strictSchemasEnabled && typeof topicOrDesc === "string") {
|
|
179
|
+
const schema = deps.schemaRegistry.get(topicOrDesc);
|
|
180
|
+
if (schema) {
|
|
181
|
+
try {
|
|
182
|
+
return await schema.parse(message);
|
|
183
|
+
} catch (error) {
|
|
184
|
+
throw new KafkaValidationError(topicName, message, {
|
|
185
|
+
cause: error instanceof Error ? error : new Error(String(error))
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return message;
|
|
191
|
+
}
|
|
192
|
+
async function buildSendPayload(topicOrDesc, messages, deps) {
|
|
193
|
+
const topic2 = resolveTopicName(topicOrDesc);
|
|
194
|
+
const builtMessages = await Promise.all(
|
|
195
|
+
messages.map(async (m) => {
|
|
196
|
+
const envelopeHeaders = buildEnvelopeHeaders({
|
|
197
|
+
correlationId: m.correlationId,
|
|
198
|
+
schemaVersion: m.schemaVersion,
|
|
199
|
+
eventId: m.eventId,
|
|
200
|
+
headers: m.headers
|
|
201
|
+
});
|
|
202
|
+
for (const inst of deps.instrumentation) {
|
|
203
|
+
inst.beforeSend?.(topic2, envelopeHeaders);
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
value: JSON.stringify(
|
|
207
|
+
await validateMessage(topicOrDesc, m.value, deps)
|
|
208
|
+
),
|
|
209
|
+
key: m.key ?? null,
|
|
210
|
+
headers: envelopeHeaders
|
|
211
|
+
};
|
|
212
|
+
})
|
|
213
|
+
);
|
|
214
|
+
return { topic: topic2, messages: builtMessages };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// src/client/kafka.client/consumer-ops.ts
|
|
218
|
+
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps) {
|
|
219
|
+
const { consumers, consumerCreationOptions, kafka, onRebalance, logger } = deps;
|
|
220
|
+
if (consumers.has(groupId)) {
|
|
221
|
+
const prev = consumerCreationOptions.get(groupId);
|
|
222
|
+
if (prev.fromBeginning !== fromBeginning || prev.autoCommit !== autoCommit) {
|
|
223
|
+
logger.warn(
|
|
224
|
+
`Consumer group "${groupId}" already exists with options (fromBeginning: ${prev.fromBeginning}, autoCommit: ${prev.autoCommit}) \u2014 new options (fromBeginning: ${fromBeginning}, autoCommit: ${autoCommit}) ignored. Use a different groupId to apply different options.`
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
return consumers.get(groupId);
|
|
228
|
+
}
|
|
229
|
+
consumerCreationOptions.set(groupId, { fromBeginning, autoCommit });
|
|
230
|
+
const config = {
|
|
231
|
+
kafkaJS: { groupId, fromBeginning, autoCommit }
|
|
232
|
+
};
|
|
233
|
+
if (onRebalance) {
|
|
234
|
+
const cb = onRebalance;
|
|
235
|
+
config["rebalance_cb"] = (err, assignment) => {
|
|
236
|
+
const type = err.code === -175 ? "assign" : "revoke";
|
|
237
|
+
try {
|
|
238
|
+
cb(
|
|
239
|
+
type,
|
|
240
|
+
assignment.map((p) => ({ topic: p.topic, partition: p.partition }))
|
|
241
|
+
);
|
|
242
|
+
} catch (e) {
|
|
243
|
+
logger.warn(`onRebalance callback threw: ${e.message}`);
|
|
244
|
+
}
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
const consumer = kafka.consumer(config);
|
|
248
|
+
consumers.set(groupId, consumer);
|
|
249
|
+
return consumer;
|
|
250
|
+
}
|
|
251
|
+
function buildSchemaMap(topics, schemaRegistry, optionSchemas) {
|
|
252
|
+
const schemaMap = /* @__PURE__ */ new Map();
|
|
253
|
+
for (const t of topics) {
|
|
254
|
+
if (t?.__schema) {
|
|
255
|
+
const name = resolveTopicName(t);
|
|
256
|
+
schemaMap.set(name, t.__schema);
|
|
257
|
+
schemaRegistry.set(name, t.__schema);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
if (optionSchemas) {
|
|
261
|
+
for (const [k, v] of optionSchemas) {
|
|
262
|
+
schemaMap.set(k, v);
|
|
263
|
+
schemaRegistry.set(k, v);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
return schemaMap;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// src/client/consumer/pipeline.ts
|
|
154
270
|
function toError(error) {
|
|
155
271
|
return error instanceof Error ? error : new Error(String(error));
|
|
156
272
|
}
|
|
@@ -237,7 +353,7 @@ var RETRY_HEADER_AFTER = "x-retry-after";
|
|
|
237
353
|
var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
|
|
238
354
|
var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
|
|
239
355
|
async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
|
|
240
|
-
const retryTopic = `${originalTopic}.retry`;
|
|
356
|
+
const retryTopic = `${originalTopic}.retry.${attempt}`;
|
|
241
357
|
const {
|
|
242
358
|
[RETRY_HEADER_ATTEMPT]: _a,
|
|
243
359
|
[RETRY_HEADER_AFTER]: _b,
|
|
@@ -269,6 +385,53 @@ async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries,
|
|
|
269
385
|
);
|
|
270
386
|
}
|
|
271
387
|
}
|
|
388
|
+
async function broadcastToInterceptors(envelopes, interceptors, cb) {
|
|
389
|
+
for (const env of envelopes) {
|
|
390
|
+
for (const interceptor of interceptors) {
|
|
391
|
+
await cb(interceptor, env);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
async function runHandlerWithPipeline(fn, envelopes, interceptors, instrumentation) {
|
|
396
|
+
const cleanups = [];
|
|
397
|
+
try {
|
|
398
|
+
for (const env of envelopes) {
|
|
399
|
+
for (const inst of instrumentation) {
|
|
400
|
+
const cleanup = inst.beforeConsume?.(env);
|
|
401
|
+
if (typeof cleanup === "function") cleanups.push(cleanup);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
for (const env of envelopes) {
|
|
405
|
+
for (const interceptor of interceptors) {
|
|
406
|
+
await interceptor.before?.(env);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
await fn();
|
|
410
|
+
for (const env of envelopes) {
|
|
411
|
+
for (const interceptor of interceptors) {
|
|
412
|
+
await interceptor.after?.(env);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
for (const cleanup of cleanups) cleanup();
|
|
416
|
+
return null;
|
|
417
|
+
} catch (error) {
|
|
418
|
+
const err = toError(error);
|
|
419
|
+
for (const env of envelopes) {
|
|
420
|
+
for (const inst of instrumentation) {
|
|
421
|
+
inst.onConsumeError?.(env, err);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
for (const cleanup of cleanups) cleanup();
|
|
425
|
+
return err;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
async function notifyInterceptorsOnError(envelopes, interceptors, error) {
|
|
429
|
+
await broadcastToInterceptors(
|
|
430
|
+
envelopes,
|
|
431
|
+
interceptors,
|
|
432
|
+
(i, env) => i.onError?.(env, error)
|
|
433
|
+
);
|
|
434
|
+
}
|
|
272
435
|
async function executeWithRetry(fn, ctx, deps) {
|
|
273
436
|
const {
|
|
274
437
|
envelope,
|
|
@@ -285,98 +448,181 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
285
448
|
const envelopes = Array.isArray(envelope) ? envelope : [envelope];
|
|
286
449
|
const topic2 = envelopes[0]?.topic ?? "unknown";
|
|
287
450
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
if (isLastAttempt && maxAttempts > 1) {
|
|
319
|
-
const exhaustedError = new KafkaRetryExhaustedError(
|
|
320
|
-
topic2,
|
|
321
|
-
envelopes.map((e) => e.payload),
|
|
322
|
-
maxAttempts,
|
|
323
|
-
{ cause: err }
|
|
324
|
-
);
|
|
325
|
-
for (const env of envelopes) {
|
|
326
|
-
for (const interceptor of interceptors) {
|
|
327
|
-
await interceptor.onError?.(env, exhaustedError);
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
} else {
|
|
331
|
-
for (const env of envelopes) {
|
|
332
|
-
for (const interceptor of interceptors) {
|
|
333
|
-
await interceptor.onError?.(env, err);
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
deps.logger.error(
|
|
338
|
-
`Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
|
|
339
|
-
err.stack
|
|
451
|
+
const error = await runHandlerWithPipeline(
|
|
452
|
+
fn,
|
|
453
|
+
envelopes,
|
|
454
|
+
interceptors,
|
|
455
|
+
deps.instrumentation
|
|
456
|
+
);
|
|
457
|
+
if (!error) return;
|
|
458
|
+
const isLastAttempt = attempt === maxAttempts;
|
|
459
|
+
const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
|
|
460
|
+
topic2,
|
|
461
|
+
envelopes.map((e) => e.payload),
|
|
462
|
+
maxAttempts,
|
|
463
|
+
{ cause: error }
|
|
464
|
+
) : error;
|
|
465
|
+
await notifyInterceptorsOnError(envelopes, interceptors, reportedError);
|
|
466
|
+
deps.logger.error(
|
|
467
|
+
`Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
|
|
468
|
+
error.stack
|
|
469
|
+
);
|
|
470
|
+
if (retryTopics && retry) {
|
|
471
|
+
const cap = Math.min(backoffMs, maxBackoffMs);
|
|
472
|
+
const delay = Math.floor(Math.random() * cap);
|
|
473
|
+
await sendToRetryTopic(
|
|
474
|
+
topic2,
|
|
475
|
+
rawMessages,
|
|
476
|
+
1,
|
|
477
|
+
retry.maxRetries,
|
|
478
|
+
delay,
|
|
479
|
+
envelopes[0]?.headers ?? {},
|
|
480
|
+
deps
|
|
340
481
|
);
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
const
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
envelopes[0]?.headers ?? {},
|
|
351
|
-
deps
|
|
352
|
-
);
|
|
353
|
-
} else if (isLastAttempt) {
|
|
354
|
-
if (dlq) {
|
|
355
|
-
const dlqMeta = {
|
|
356
|
-
error: err,
|
|
357
|
-
attempt,
|
|
358
|
-
originalHeaders: envelopes[0]?.headers
|
|
359
|
-
};
|
|
360
|
-
for (const raw of rawMessages) {
|
|
361
|
-
await sendToDlq(topic2, raw, deps, dlqMeta);
|
|
362
|
-
}
|
|
363
|
-
} else {
|
|
364
|
-
await deps.onMessageLost?.({
|
|
365
|
-
topic: topic2,
|
|
366
|
-
error: err,
|
|
367
|
-
attempt,
|
|
368
|
-
headers: envelopes[0]?.headers ?? {}
|
|
369
|
-
});
|
|
482
|
+
} else if (isLastAttempt) {
|
|
483
|
+
if (dlq) {
|
|
484
|
+
const dlqMeta = {
|
|
485
|
+
error,
|
|
486
|
+
attempt,
|
|
487
|
+
originalHeaders: envelopes[0]?.headers
|
|
488
|
+
};
|
|
489
|
+
for (const raw of rawMessages) {
|
|
490
|
+
await sendToDlq(topic2, raw, deps, dlqMeta);
|
|
370
491
|
}
|
|
371
492
|
} else {
|
|
372
|
-
|
|
373
|
-
|
|
493
|
+
await deps.onMessageLost?.({
|
|
494
|
+
topic: topic2,
|
|
495
|
+
error,
|
|
496
|
+
attempt,
|
|
497
|
+
headers: envelopes[0]?.headers ?? {}
|
|
498
|
+
});
|
|
374
499
|
}
|
|
500
|
+
} else {
|
|
501
|
+
const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
|
|
502
|
+
await sleep(Math.random() * cap);
|
|
375
503
|
}
|
|
376
504
|
}
|
|
377
505
|
}
|
|
378
506
|
|
|
379
|
-
// src/client/
|
|
507
|
+
// src/client/kafka.client/message-handler.ts
|
|
508
|
+
async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
|
|
509
|
+
if (!message.value) {
|
|
510
|
+
deps.logger.warn(`Received empty message from topic ${topic2}`);
|
|
511
|
+
return null;
|
|
512
|
+
}
|
|
513
|
+
const raw = message.value.toString();
|
|
514
|
+
const parsed = parseJsonMessage(raw, topic2, deps.logger);
|
|
515
|
+
if (parsed === null) return null;
|
|
516
|
+
const headers = decodeHeaders(message.headers);
|
|
517
|
+
const validated = await validateWithSchema(
|
|
518
|
+
parsed,
|
|
519
|
+
raw,
|
|
520
|
+
topic2,
|
|
521
|
+
schemaMap,
|
|
522
|
+
interceptors,
|
|
523
|
+
dlq,
|
|
524
|
+
{ ...deps, originalHeaders: headers }
|
|
525
|
+
);
|
|
526
|
+
if (validated === null) return null;
|
|
527
|
+
return extractEnvelope(validated, headers, topic2, partition, message.offset);
|
|
528
|
+
}
|
|
529
|
+
async function handleEachMessage(payload, opts, deps) {
|
|
530
|
+
const { topic: topic2, partition, message } = payload;
|
|
531
|
+
const {
|
|
532
|
+
schemaMap,
|
|
533
|
+
handleMessage,
|
|
534
|
+
interceptors,
|
|
535
|
+
dlq,
|
|
536
|
+
retry,
|
|
537
|
+
retryTopics,
|
|
538
|
+
timeoutMs,
|
|
539
|
+
wrapWithTimeout
|
|
540
|
+
} = opts;
|
|
541
|
+
const envelope = await parseSingleMessage(
|
|
542
|
+
message,
|
|
543
|
+
topic2,
|
|
544
|
+
partition,
|
|
545
|
+
schemaMap,
|
|
546
|
+
interceptors,
|
|
547
|
+
dlq,
|
|
548
|
+
deps
|
|
549
|
+
);
|
|
550
|
+
if (envelope === null) return;
|
|
551
|
+
await executeWithRetry(
|
|
552
|
+
() => {
|
|
553
|
+
const fn = () => runWithEnvelopeContext(
|
|
554
|
+
{
|
|
555
|
+
correlationId: envelope.correlationId,
|
|
556
|
+
traceparent: envelope.traceparent
|
|
557
|
+
},
|
|
558
|
+
() => handleMessage(envelope)
|
|
559
|
+
);
|
|
560
|
+
return timeoutMs ? wrapWithTimeout(fn, timeoutMs, topic2) : fn();
|
|
561
|
+
},
|
|
562
|
+
{
|
|
563
|
+
envelope,
|
|
564
|
+
rawMessages: [message.value.toString()],
|
|
565
|
+
interceptors,
|
|
566
|
+
dlq,
|
|
567
|
+
retry,
|
|
568
|
+
retryTopics
|
|
569
|
+
},
|
|
570
|
+
deps
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
async function handleEachBatch(payload, opts, deps) {
|
|
574
|
+
const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
|
|
575
|
+
const {
|
|
576
|
+
schemaMap,
|
|
577
|
+
handleBatch,
|
|
578
|
+
interceptors,
|
|
579
|
+
dlq,
|
|
580
|
+
retry,
|
|
581
|
+
timeoutMs,
|
|
582
|
+
wrapWithTimeout
|
|
583
|
+
} = opts;
|
|
584
|
+
const envelopes = [];
|
|
585
|
+
const rawMessages = [];
|
|
586
|
+
for (const message of batch.messages) {
|
|
587
|
+
const envelope = await parseSingleMessage(
|
|
588
|
+
message,
|
|
589
|
+
batch.topic,
|
|
590
|
+
batch.partition,
|
|
591
|
+
schemaMap,
|
|
592
|
+
interceptors,
|
|
593
|
+
dlq,
|
|
594
|
+
deps
|
|
595
|
+
);
|
|
596
|
+
if (envelope === null) continue;
|
|
597
|
+
envelopes.push(envelope);
|
|
598
|
+
rawMessages.push(message.value.toString());
|
|
599
|
+
}
|
|
600
|
+
if (envelopes.length === 0) return;
|
|
601
|
+
const meta = {
|
|
602
|
+
partition: batch.partition,
|
|
603
|
+
highWatermark: batch.highWatermark,
|
|
604
|
+
heartbeat,
|
|
605
|
+
resolveOffset,
|
|
606
|
+
commitOffsetsIfNecessary
|
|
607
|
+
};
|
|
608
|
+
await executeWithRetry(
|
|
609
|
+
() => {
|
|
610
|
+
const fn = () => handleBatch(envelopes, meta);
|
|
611
|
+
return timeoutMs ? wrapWithTimeout(fn, timeoutMs, batch.topic) : fn();
|
|
612
|
+
},
|
|
613
|
+
{
|
|
614
|
+
envelope: envelopes,
|
|
615
|
+
rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
|
|
616
|
+
interceptors,
|
|
617
|
+
dlq,
|
|
618
|
+
retry,
|
|
619
|
+
isBatch: true
|
|
620
|
+
},
|
|
621
|
+
deps
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// src/client/consumer/subscribe-retry.ts
|
|
380
626
|
async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
381
627
|
const maxAttempts = retryOpts?.retries ?? 5;
|
|
382
628
|
const backoffMs = retryOpts?.backoffMs ?? 5e3;
|
|
@@ -395,7 +641,183 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
|
395
641
|
}
|
|
396
642
|
}
|
|
397
643
|
|
|
398
|
-
// src/client/kafka.client.ts
|
|
644
|
+
// src/client/kafka.client/retry-topic.ts
|
|
645
|
+
async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs = 1e4) {
|
|
646
|
+
const topicSet = new Set(topics);
|
|
647
|
+
const deadline = Date.now() + timeoutMs;
|
|
648
|
+
while (Date.now() < deadline) {
|
|
649
|
+
try {
|
|
650
|
+
const assigned = consumer.assignment();
|
|
651
|
+
if (assigned.some((a) => topicSet.has(a.topic))) return;
|
|
652
|
+
} catch {
|
|
653
|
+
}
|
|
654
|
+
await sleep(200);
|
|
655
|
+
}
|
|
656
|
+
logger.warn(
|
|
657
|
+
`Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
|
|
658
|
+
);
|
|
659
|
+
}
|
|
660
|
+
async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
661
|
+
const {
|
|
662
|
+
logger,
|
|
663
|
+
producer,
|
|
664
|
+
instrumentation,
|
|
665
|
+
onMessageLost,
|
|
666
|
+
ensureTopic,
|
|
667
|
+
getOrCreateConsumer: getOrCreateConsumer2,
|
|
668
|
+
runningConsumers
|
|
669
|
+
} = deps;
|
|
670
|
+
const backoffMs = retry.backoffMs ?? 1e3;
|
|
671
|
+
const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
|
|
672
|
+
const pipelineDeps = { logger, producer, instrumentation, onMessageLost };
|
|
673
|
+
for (const lt of levelTopics) {
|
|
674
|
+
await ensureTopic(lt);
|
|
675
|
+
}
|
|
676
|
+
const consumer = getOrCreateConsumer2(levelGroupId, false, false);
|
|
677
|
+
await consumer.connect();
|
|
678
|
+
await subscribeWithRetry(consumer, levelTopics, logger);
|
|
679
|
+
await consumer.run({
|
|
680
|
+
eachMessage: async ({ topic: levelTopic, partition, message }) => {
|
|
681
|
+
const nextOffset = {
|
|
682
|
+
topic: levelTopic,
|
|
683
|
+
partition,
|
|
684
|
+
offset: (parseInt(message.offset, 10) + 1).toString()
|
|
685
|
+
};
|
|
686
|
+
if (!message.value) {
|
|
687
|
+
await consumer.commitOffsets([nextOffset]);
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
const headers = decodeHeaders(message.headers);
|
|
691
|
+
const retryAfter = parseInt(
|
|
692
|
+
headers[RETRY_HEADER_AFTER] ?? "0",
|
|
693
|
+
10
|
|
694
|
+
);
|
|
695
|
+
const remaining = retryAfter - Date.now();
|
|
696
|
+
if (remaining > 0) {
|
|
697
|
+
consumer.pause([{ topic: levelTopic, partitions: [partition] }]);
|
|
698
|
+
await sleep(remaining);
|
|
699
|
+
consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
|
|
700
|
+
}
|
|
701
|
+
const raw = message.value.toString();
|
|
702
|
+
const parsed = parseJsonMessage(raw, levelTopic, logger);
|
|
703
|
+
if (parsed === null) {
|
|
704
|
+
await consumer.commitOffsets([nextOffset]);
|
|
705
|
+
return;
|
|
706
|
+
}
|
|
707
|
+
const currentMaxRetries = parseInt(
|
|
708
|
+
headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
|
|
709
|
+
10
|
|
710
|
+
);
|
|
711
|
+
const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
|
|
712
|
+
const validated = await validateWithSchema(
|
|
713
|
+
parsed,
|
|
714
|
+
raw,
|
|
715
|
+
originalTopic,
|
|
716
|
+
schemaMap,
|
|
717
|
+
interceptors,
|
|
718
|
+
dlq,
|
|
719
|
+
{ ...pipelineDeps, originalHeaders: headers }
|
|
720
|
+
);
|
|
721
|
+
if (validated === null) {
|
|
722
|
+
await consumer.commitOffsets([nextOffset]);
|
|
723
|
+
return;
|
|
724
|
+
}
|
|
725
|
+
const envelope = extractEnvelope(
|
|
726
|
+
validated,
|
|
727
|
+
headers,
|
|
728
|
+
originalTopic,
|
|
729
|
+
partition,
|
|
730
|
+
message.offset
|
|
731
|
+
);
|
|
732
|
+
const error = await runHandlerWithPipeline(
|
|
733
|
+
() => runWithEnvelopeContext(
|
|
734
|
+
{
|
|
735
|
+
correlationId: envelope.correlationId,
|
|
736
|
+
traceparent: envelope.traceparent
|
|
737
|
+
},
|
|
738
|
+
() => handleMessage(envelope)
|
|
739
|
+
),
|
|
740
|
+
[envelope],
|
|
741
|
+
interceptors,
|
|
742
|
+
instrumentation
|
|
743
|
+
);
|
|
744
|
+
if (!error) {
|
|
745
|
+
await consumer.commitOffsets([nextOffset]);
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
const exhausted = level >= currentMaxRetries;
|
|
749
|
+
const reportedError = exhausted && currentMaxRetries > 1 ? new KafkaRetryExhaustedError(
|
|
750
|
+
originalTopic,
|
|
751
|
+
[envelope.payload],
|
|
752
|
+
currentMaxRetries,
|
|
753
|
+
{ cause: error }
|
|
754
|
+
) : error;
|
|
755
|
+
await notifyInterceptorsOnError([envelope], interceptors, reportedError);
|
|
756
|
+
logger.error(
|
|
757
|
+
`Retry consumer error for ${originalTopic} (level ${level}/${currentMaxRetries}):`,
|
|
758
|
+
error.stack
|
|
759
|
+
);
|
|
760
|
+
if (!exhausted) {
|
|
761
|
+
const nextLevel = level + 1;
|
|
762
|
+
const cap = Math.min(backoffMs * 2 ** level, maxBackoffMs);
|
|
763
|
+
const delay = Math.floor(Math.random() * cap);
|
|
764
|
+
await sendToRetryTopic(
|
|
765
|
+
originalTopic,
|
|
766
|
+
[raw],
|
|
767
|
+
nextLevel,
|
|
768
|
+
currentMaxRetries,
|
|
769
|
+
delay,
|
|
770
|
+
headers,
|
|
771
|
+
pipelineDeps
|
|
772
|
+
);
|
|
773
|
+
} else if (dlq) {
|
|
774
|
+
await sendToDlq(originalTopic, raw, pipelineDeps, {
|
|
775
|
+
error,
|
|
776
|
+
// +1 to account for the main consumer's initial attempt before routing.
|
|
777
|
+
attempt: level + 1,
|
|
778
|
+
originalHeaders: headers
|
|
779
|
+
});
|
|
780
|
+
} else {
|
|
781
|
+
await onMessageLost?.({
|
|
782
|
+
topic: originalTopic,
|
|
783
|
+
error,
|
|
784
|
+
attempt: level,
|
|
785
|
+
headers
|
|
786
|
+
});
|
|
787
|
+
}
|
|
788
|
+
await consumer.commitOffsets([nextOffset]);
|
|
789
|
+
}
|
|
790
|
+
});
|
|
791
|
+
runningConsumers.set(levelGroupId, "eachMessage");
|
|
792
|
+
await waitForPartitionAssignment(consumer, levelTopics, logger, assignmentTimeoutMs);
|
|
793
|
+
logger.log(
|
|
794
|
+
`Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
|
|
795
|
+
);
|
|
796
|
+
}
|
|
797
|
+
async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
798
|
+
const levelGroupIds = [];
|
|
799
|
+
for (let level = 1; level <= retry.maxRetries; level++) {
|
|
800
|
+
const levelTopics = originalTopics.map((t) => `${t}.retry.${level}`);
|
|
801
|
+
const levelGroupId = `${originalGroupId}-retry.${level}`;
|
|
802
|
+
await startLevelConsumer(
|
|
803
|
+
level,
|
|
804
|
+
levelTopics,
|
|
805
|
+
levelGroupId,
|
|
806
|
+
originalTopics,
|
|
807
|
+
handleMessage,
|
|
808
|
+
retry,
|
|
809
|
+
dlq,
|
|
810
|
+
interceptors,
|
|
811
|
+
schemaMap,
|
|
812
|
+
deps,
|
|
813
|
+
assignmentTimeoutMs
|
|
814
|
+
);
|
|
815
|
+
levelGroupIds.push(levelGroupId);
|
|
816
|
+
}
|
|
817
|
+
return levelGroupIds;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
// src/client/kafka.client/index.ts
|
|
399
821
|
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = import_kafka_javascript.KafkaJS;
|
|
400
822
|
var KafkaClient = class {
|
|
401
823
|
kafka;
|
|
@@ -411,6 +833,9 @@ var KafkaClient = class {
|
|
|
411
833
|
defaultGroupId;
|
|
412
834
|
schemaRegistry = /* @__PURE__ */ new Map();
|
|
413
835
|
runningConsumers = /* @__PURE__ */ new Map();
|
|
836
|
+
consumerCreationOptions = /* @__PURE__ */ new Map();
|
|
837
|
+
/** Maps each main consumer groupId to its companion retry level groupIds. */
|
|
838
|
+
companionGroupIds = /* @__PURE__ */ new Map();
|
|
414
839
|
instrumentation;
|
|
415
840
|
onMessageLost;
|
|
416
841
|
onRebalance;
|
|
@@ -445,7 +870,7 @@ var KafkaClient = class {
|
|
|
445
870
|
this.admin = this.kafka.admin();
|
|
446
871
|
}
|
|
447
872
|
async sendMessage(topicOrDesc, message, options = {}) {
|
|
448
|
-
const payload = await this.
|
|
873
|
+
const payload = await this.preparePayload(topicOrDesc, [
|
|
449
874
|
{
|
|
450
875
|
value: message,
|
|
451
876
|
key: options.key,
|
|
@@ -455,19 +880,13 @@ var KafkaClient = class {
|
|
|
455
880
|
eventId: options.eventId
|
|
456
881
|
}
|
|
457
882
|
]);
|
|
458
|
-
await this.ensureTopic(payload.topic);
|
|
459
883
|
await this.producer.send(payload);
|
|
460
|
-
|
|
461
|
-
inst.afterSend?.(payload.topic);
|
|
462
|
-
}
|
|
884
|
+
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
463
885
|
}
|
|
464
886
|
async sendBatch(topicOrDesc, messages) {
|
|
465
|
-
const payload = await this.
|
|
466
|
-
await this.ensureTopic(payload.topic);
|
|
887
|
+
const payload = await this.preparePayload(topicOrDesc, messages);
|
|
467
888
|
await this.producer.send(payload);
|
|
468
|
-
|
|
469
|
-
inst.afterSend?.(payload.topic);
|
|
470
|
-
}
|
|
889
|
+
this.notifyAfterSend(payload.topic, payload.messages.length);
|
|
471
890
|
}
|
|
472
891
|
/** Execute multiple sends atomically. Commits on success, aborts on error. */
|
|
473
892
|
async transaction(fn) {
|
|
@@ -486,7 +905,7 @@ var KafkaClient = class {
|
|
|
486
905
|
try {
|
|
487
906
|
const ctx = {
|
|
488
907
|
send: async (topicOrDesc, message, options = {}) => {
|
|
489
|
-
const payload = await this.
|
|
908
|
+
const payload = await this.preparePayload(topicOrDesc, [
|
|
490
909
|
{
|
|
491
910
|
value: message,
|
|
492
911
|
key: options.key,
|
|
@@ -496,13 +915,10 @@ var KafkaClient = class {
|
|
|
496
915
|
eventId: options.eventId
|
|
497
916
|
}
|
|
498
917
|
]);
|
|
499
|
-
await this.ensureTopic(payload.topic);
|
|
500
918
|
await tx.send(payload);
|
|
501
919
|
},
|
|
502
920
|
sendBatch: async (topicOrDesc, messages) => {
|
|
503
|
-
|
|
504
|
-
await this.ensureTopic(payload.topic);
|
|
505
|
-
await tx.send(payload);
|
|
921
|
+
await tx.send(await this.preparePayload(topicOrDesc, messages));
|
|
506
922
|
}
|
|
507
923
|
};
|
|
508
924
|
await fn(ctx);
|
|
@@ -536,151 +952,59 @@ var KafkaClient = class {
|
|
|
536
952
|
);
|
|
537
953
|
}
|
|
538
954
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
|
|
539
|
-
const deps =
|
|
540
|
-
logger: this.logger,
|
|
541
|
-
producer: this.producer,
|
|
542
|
-
instrumentation: this.instrumentation,
|
|
543
|
-
onMessageLost: this.onMessageLost
|
|
544
|
-
};
|
|
955
|
+
const deps = this.messageDeps;
|
|
545
956
|
const timeoutMs = options.handlerTimeoutMs;
|
|
546
957
|
await consumer.run({
|
|
547
|
-
eachMessage:
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
return;
|
|
551
|
-
}
|
|
552
|
-
const raw = message.value.toString();
|
|
553
|
-
const parsed = parseJsonMessage(raw, topic2, this.logger);
|
|
554
|
-
if (parsed === null) return;
|
|
555
|
-
const headers = decodeHeaders(message.headers);
|
|
556
|
-
const validated = await validateWithSchema(
|
|
557
|
-
parsed,
|
|
558
|
-
raw,
|
|
559
|
-
topic2,
|
|
958
|
+
eachMessage: (payload) => handleEachMessage(
|
|
959
|
+
payload,
|
|
960
|
+
{
|
|
560
961
|
schemaMap,
|
|
962
|
+
handleMessage,
|
|
561
963
|
interceptors,
|
|
562
964
|
dlq,
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
partition,
|
|
571
|
-
message.offset
|
|
572
|
-
);
|
|
573
|
-
await executeWithRetry(
|
|
574
|
-
() => {
|
|
575
|
-
const fn = () => runWithEnvelopeContext(
|
|
576
|
-
{
|
|
577
|
-
correlationId: envelope.correlationId,
|
|
578
|
-
traceparent: envelope.traceparent
|
|
579
|
-
},
|
|
580
|
-
() => handleMessage(envelope)
|
|
581
|
-
);
|
|
582
|
-
return timeoutMs ? this.wrapWithTimeoutWarning(fn, timeoutMs, topic2) : fn();
|
|
583
|
-
},
|
|
584
|
-
{
|
|
585
|
-
envelope,
|
|
586
|
-
rawMessages: [raw],
|
|
587
|
-
interceptors,
|
|
588
|
-
dlq,
|
|
589
|
-
retry,
|
|
590
|
-
retryTopics: options.retryTopics
|
|
591
|
-
},
|
|
592
|
-
deps
|
|
593
|
-
);
|
|
594
|
-
}
|
|
965
|
+
retry,
|
|
966
|
+
retryTopics: options.retryTopics,
|
|
967
|
+
timeoutMs,
|
|
968
|
+
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
|
|
969
|
+
},
|
|
970
|
+
deps
|
|
971
|
+
)
|
|
595
972
|
});
|
|
596
973
|
this.runningConsumers.set(gid, "eachMessage");
|
|
597
974
|
if (options.retryTopics && retry) {
|
|
598
|
-
await
|
|
975
|
+
const companions = await startRetryTopicConsumers(
|
|
599
976
|
topicNames,
|
|
600
977
|
gid,
|
|
601
978
|
handleMessage,
|
|
602
979
|
retry,
|
|
603
980
|
dlq,
|
|
604
981
|
interceptors,
|
|
605
|
-
schemaMap
|
|
982
|
+
schemaMap,
|
|
983
|
+
this.retryTopicDeps,
|
|
984
|
+
options.retryTopicAssignmentTimeoutMs
|
|
606
985
|
);
|
|
986
|
+
this.companionGroupIds.set(gid, companions);
|
|
607
987
|
}
|
|
608
988
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
609
989
|
}
|
|
610
990
|
async startBatchConsumer(topics, handleBatch, options = {}) {
|
|
611
991
|
const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
|
|
612
|
-
const deps =
|
|
613
|
-
logger: this.logger,
|
|
614
|
-
producer: this.producer,
|
|
615
|
-
instrumentation: this.instrumentation,
|
|
616
|
-
onMessageLost: this.onMessageLost
|
|
617
|
-
};
|
|
992
|
+
const deps = this.messageDeps;
|
|
618
993
|
const timeoutMs = options.handlerTimeoutMs;
|
|
619
994
|
await consumer.run({
|
|
620
|
-
eachBatch:
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
continue;
|
|
634
|
-
}
|
|
635
|
-
const raw = message.value.toString();
|
|
636
|
-
const parsed = parseJsonMessage(raw, batch.topic, this.logger);
|
|
637
|
-
if (parsed === null) continue;
|
|
638
|
-
const headers = decodeHeaders(message.headers);
|
|
639
|
-
const validated = await validateWithSchema(
|
|
640
|
-
parsed,
|
|
641
|
-
raw,
|
|
642
|
-
batch.topic,
|
|
643
|
-
schemaMap,
|
|
644
|
-
interceptors,
|
|
645
|
-
dlq,
|
|
646
|
-
{ ...deps, originalHeaders: headers }
|
|
647
|
-
);
|
|
648
|
-
if (validated === null) continue;
|
|
649
|
-
envelopes.push(
|
|
650
|
-
extractEnvelope(
|
|
651
|
-
validated,
|
|
652
|
-
headers,
|
|
653
|
-
batch.topic,
|
|
654
|
-
batch.partition,
|
|
655
|
-
message.offset
|
|
656
|
-
)
|
|
657
|
-
);
|
|
658
|
-
rawMessages.push(raw);
|
|
659
|
-
}
|
|
660
|
-
if (envelopes.length === 0) return;
|
|
661
|
-
const meta = {
|
|
662
|
-
partition: batch.partition,
|
|
663
|
-
highWatermark: batch.highWatermark,
|
|
664
|
-
heartbeat,
|
|
665
|
-
resolveOffset,
|
|
666
|
-
commitOffsetsIfNecessary
|
|
667
|
-
};
|
|
668
|
-
await executeWithRetry(
|
|
669
|
-
() => {
|
|
670
|
-
const fn = () => handleBatch(envelopes, meta);
|
|
671
|
-
return timeoutMs ? this.wrapWithTimeoutWarning(fn, timeoutMs, batch.topic) : fn();
|
|
672
|
-
},
|
|
673
|
-
{
|
|
674
|
-
envelope: envelopes,
|
|
675
|
-
rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
|
|
676
|
-
interceptors,
|
|
677
|
-
dlq,
|
|
678
|
-
retry,
|
|
679
|
-
isBatch: true
|
|
680
|
-
},
|
|
681
|
-
deps
|
|
682
|
-
);
|
|
683
|
-
}
|
|
995
|
+
eachBatch: (payload) => handleEachBatch(
|
|
996
|
+
payload,
|
|
997
|
+
{
|
|
998
|
+
schemaMap,
|
|
999
|
+
handleBatch,
|
|
1000
|
+
interceptors,
|
|
1001
|
+
dlq,
|
|
1002
|
+
retry,
|
|
1003
|
+
timeoutMs,
|
|
1004
|
+
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
|
|
1005
|
+
},
|
|
1006
|
+
deps
|
|
1007
|
+
)
|
|
684
1008
|
});
|
|
685
1009
|
this.runningConsumers.set(gid, "eachBatch");
|
|
686
1010
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
@@ -699,7 +1023,21 @@ var KafkaClient = class {
|
|
|
699
1023
|
});
|
|
700
1024
|
this.consumers.delete(groupId);
|
|
701
1025
|
this.runningConsumers.delete(groupId);
|
|
1026
|
+
this.consumerCreationOptions.delete(groupId);
|
|
702
1027
|
this.logger.log(`Consumer disconnected: group "${groupId}"`);
|
|
1028
|
+
const companions = this.companionGroupIds.get(groupId) ?? [];
|
|
1029
|
+
for (const cGroupId of companions) {
|
|
1030
|
+
const cConsumer = this.consumers.get(cGroupId);
|
|
1031
|
+
if (cConsumer) {
|
|
1032
|
+
await cConsumer.disconnect().catch(() => {
|
|
1033
|
+
});
|
|
1034
|
+
this.consumers.delete(cGroupId);
|
|
1035
|
+
this.runningConsumers.delete(cGroupId);
|
|
1036
|
+
this.consumerCreationOptions.delete(cGroupId);
|
|
1037
|
+
this.logger.log(`Retry consumer disconnected: group "${cGroupId}"`);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
this.companionGroupIds.delete(groupId);
|
|
703
1041
|
} else {
|
|
704
1042
|
const tasks = Array.from(this.consumers.values()).map(
|
|
705
1043
|
(c) => c.disconnect().catch(() => {
|
|
@@ -708,6 +1046,8 @@ var KafkaClient = class {
|
|
|
708
1046
|
await Promise.allSettled(tasks);
|
|
709
1047
|
this.consumers.clear();
|
|
710
1048
|
this.runningConsumers.clear();
|
|
1049
|
+
this.consumerCreationOptions.clear();
|
|
1050
|
+
this.companionGroupIds.clear();
|
|
711
1051
|
this.logger.log("All consumers disconnected");
|
|
712
1052
|
}
|
|
713
1053
|
}
|
|
@@ -737,14 +1077,22 @@ var KafkaClient = class {
|
|
|
737
1077
|
}
|
|
738
1078
|
return result;
|
|
739
1079
|
}
|
|
740
|
-
/** Check broker connectivity
|
|
1080
|
+
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
741
1081
|
async checkStatus() {
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
1082
|
+
try {
|
|
1083
|
+
if (!this.isAdminConnected) {
|
|
1084
|
+
await this.admin.connect();
|
|
1085
|
+
this.isAdminConnected = true;
|
|
1086
|
+
}
|
|
1087
|
+
const topics = await this.admin.listTopics();
|
|
1088
|
+
return { status: "up", clientId: this.clientId, topics };
|
|
1089
|
+
} catch (error) {
|
|
1090
|
+
return {
|
|
1091
|
+
status: "down",
|
|
1092
|
+
clientId: this.clientId,
|
|
1093
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1094
|
+
};
|
|
745
1095
|
}
|
|
746
|
-
const topics = await this.admin.listTopics();
|
|
747
|
-
return { status: "up", clientId: this.clientId, topics };
|
|
748
1096
|
}
|
|
749
1097
|
getClientId() {
|
|
750
1098
|
return this.clientId;
|
|
@@ -766,204 +1114,28 @@ var KafkaClient = class {
|
|
|
766
1114
|
await Promise.allSettled(tasks);
|
|
767
1115
|
this.consumers.clear();
|
|
768
1116
|
this.runningConsumers.clear();
|
|
1117
|
+
this.consumerCreationOptions.clear();
|
|
1118
|
+
this.companionGroupIds.clear();
|
|
769
1119
|
this.logger.log("All connections closed");
|
|
770
1120
|
}
|
|
771
|
-
// ── Retry topic chain ────────────────────────────────────────────
|
|
772
|
-
/**
|
|
773
|
-
* Auto-start companion consumers on `<topic>.retry` for each original topic.
|
|
774
|
-
* Called by `startConsumer` when `retryTopics: true`.
|
|
775
|
-
*
|
|
776
|
-
* Flow per message:
|
|
777
|
-
* 1. Sleep until `x-retry-after` (scheduled by the main consumer or previous retry hop)
|
|
778
|
-
* 2. Call the original handler
|
|
779
|
-
* 3. On failure: if retries remain → re-send to `<originalTopic>.retry` with incremented attempt
|
|
780
|
-
* if exhausted → DLQ or onMessageLost
|
|
781
|
-
*/
|
|
782
|
-
async startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap) {
|
|
783
|
-
const retryTopicNames = originalTopics.map((t) => `${t}.retry`);
|
|
784
|
-
const retryGroupId = `${originalGroupId}-retry`;
|
|
785
|
-
const backoffMs = retry.backoffMs ?? 1e3;
|
|
786
|
-
const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
|
|
787
|
-
const deps = {
|
|
788
|
-
logger: this.logger,
|
|
789
|
-
producer: this.producer,
|
|
790
|
-
instrumentation: this.instrumentation,
|
|
791
|
-
onMessageLost: this.onMessageLost
|
|
792
|
-
};
|
|
793
|
-
for (const rt of retryTopicNames) {
|
|
794
|
-
await this.ensureTopic(rt);
|
|
795
|
-
}
|
|
796
|
-
const consumer = this.getOrCreateConsumer(retryGroupId, false, true);
|
|
797
|
-
await consumer.connect();
|
|
798
|
-
await subscribeWithRetry(consumer, retryTopicNames, this.logger);
|
|
799
|
-
await consumer.run({
|
|
800
|
-
eachMessage: async ({ topic: retryTopic, partition, message }) => {
|
|
801
|
-
if (!message.value) return;
|
|
802
|
-
const raw = message.value.toString();
|
|
803
|
-
const parsed = parseJsonMessage(raw, retryTopic, this.logger);
|
|
804
|
-
if (parsed === null) return;
|
|
805
|
-
const headers = decodeHeaders(message.headers);
|
|
806
|
-
const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? retryTopic.replace(/\.retry$/, "");
|
|
807
|
-
const currentAttempt = parseInt(
|
|
808
|
-
headers[RETRY_HEADER_ATTEMPT] ?? "1",
|
|
809
|
-
10
|
|
810
|
-
);
|
|
811
|
-
const maxRetries = parseInt(
|
|
812
|
-
headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
|
|
813
|
-
10
|
|
814
|
-
);
|
|
815
|
-
const retryAfter = parseInt(
|
|
816
|
-
headers[RETRY_HEADER_AFTER] ?? "0",
|
|
817
|
-
10
|
|
818
|
-
);
|
|
819
|
-
const remaining = retryAfter - Date.now();
|
|
820
|
-
if (remaining > 0) {
|
|
821
|
-
consumer.pause([{ topic: retryTopic, partitions: [partition] }]);
|
|
822
|
-
await sleep(remaining);
|
|
823
|
-
consumer.resume([{ topic: retryTopic, partitions: [partition] }]);
|
|
824
|
-
}
|
|
825
|
-
const validated = await validateWithSchema(
|
|
826
|
-
parsed,
|
|
827
|
-
raw,
|
|
828
|
-
originalTopic,
|
|
829
|
-
schemaMap,
|
|
830
|
-
interceptors,
|
|
831
|
-
dlq,
|
|
832
|
-
{ ...deps, originalHeaders: headers }
|
|
833
|
-
);
|
|
834
|
-
if (validated === null) return;
|
|
835
|
-
const envelope = extractEnvelope(
|
|
836
|
-
validated,
|
|
837
|
-
headers,
|
|
838
|
-
originalTopic,
|
|
839
|
-
partition,
|
|
840
|
-
message.offset
|
|
841
|
-
);
|
|
842
|
-
try {
|
|
843
|
-
const cleanups = [];
|
|
844
|
-
for (const inst of this.instrumentation) {
|
|
845
|
-
const c = inst.beforeConsume?.(envelope);
|
|
846
|
-
if (typeof c === "function") cleanups.push(c);
|
|
847
|
-
}
|
|
848
|
-
for (const interceptor of interceptors)
|
|
849
|
-
await interceptor.before?.(envelope);
|
|
850
|
-
await runWithEnvelopeContext(
|
|
851
|
-
{
|
|
852
|
-
correlationId: envelope.correlationId,
|
|
853
|
-
traceparent: envelope.traceparent
|
|
854
|
-
},
|
|
855
|
-
() => handleMessage(envelope)
|
|
856
|
-
);
|
|
857
|
-
for (const interceptor of interceptors)
|
|
858
|
-
await interceptor.after?.(envelope);
|
|
859
|
-
for (const cleanup of cleanups) cleanup();
|
|
860
|
-
} catch (error) {
|
|
861
|
-
const err = toError(error);
|
|
862
|
-
const nextAttempt = currentAttempt + 1;
|
|
863
|
-
const exhausted = currentAttempt >= maxRetries;
|
|
864
|
-
for (const inst of this.instrumentation)
|
|
865
|
-
inst.onConsumeError?.(envelope, err);
|
|
866
|
-
const reportedError = exhausted && maxRetries > 1 ? new KafkaRetryExhaustedError(
|
|
867
|
-
originalTopic,
|
|
868
|
-
[envelope.payload],
|
|
869
|
-
maxRetries,
|
|
870
|
-
{ cause: err }
|
|
871
|
-
) : err;
|
|
872
|
-
for (const interceptor of interceptors) {
|
|
873
|
-
await interceptor.onError?.(envelope, reportedError);
|
|
874
|
-
}
|
|
875
|
-
this.logger.error(
|
|
876
|
-
`Retry consumer error for ${originalTopic} (attempt ${currentAttempt}/${maxRetries}):`,
|
|
877
|
-
err.stack
|
|
878
|
-
);
|
|
879
|
-
if (!exhausted) {
|
|
880
|
-
const cap = Math.min(backoffMs * 2 ** currentAttempt, maxBackoffMs);
|
|
881
|
-
const delay = Math.floor(Math.random() * cap);
|
|
882
|
-
await sendToRetryTopic(
|
|
883
|
-
originalTopic,
|
|
884
|
-
[raw],
|
|
885
|
-
nextAttempt,
|
|
886
|
-
maxRetries,
|
|
887
|
-
delay,
|
|
888
|
-
headers,
|
|
889
|
-
deps
|
|
890
|
-
);
|
|
891
|
-
} else if (dlq) {
|
|
892
|
-
await sendToDlq(originalTopic, raw, deps, {
|
|
893
|
-
error: err,
|
|
894
|
-
// +1 to account for the main consumer's initial attempt before
|
|
895
|
-
// routing to the retry topic, making this consistent with the
|
|
896
|
-
// in-process retry path where attempt counts all tries.
|
|
897
|
-
attempt: currentAttempt + 1,
|
|
898
|
-
originalHeaders: headers
|
|
899
|
-
});
|
|
900
|
-
} else {
|
|
901
|
-
await deps.onMessageLost?.({
|
|
902
|
-
topic: originalTopic,
|
|
903
|
-
error: err,
|
|
904
|
-
attempt: currentAttempt,
|
|
905
|
-
headers
|
|
906
|
-
});
|
|
907
|
-
}
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
});
|
|
911
|
-
this.runningConsumers.set(retryGroupId, "eachMessage");
|
|
912
|
-
await this.waitForPartitionAssignment(consumer, retryTopicNames);
|
|
913
|
-
this.logger.log(
|
|
914
|
-
`Retry topic consumers started for: ${originalTopics.join(", ")} (group: ${retryGroupId})`
|
|
915
|
-
);
|
|
916
|
-
}
|
|
917
1121
|
// ── Private helpers ──────────────────────────────────────────────
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
*/
|
|
925
|
-
async waitForPartitionAssignment(consumer, topics, timeoutMs = 1e4) {
|
|
926
|
-
const topicSet = new Set(topics);
|
|
927
|
-
const deadline = Date.now() + timeoutMs;
|
|
928
|
-
while (Date.now() < deadline) {
|
|
929
|
-
try {
|
|
930
|
-
const assigned = consumer.assignment();
|
|
931
|
-
if (assigned.some((a) => topicSet.has(a.topic))) return;
|
|
932
|
-
} catch {
|
|
933
|
-
}
|
|
934
|
-
await sleep(200);
|
|
935
|
-
}
|
|
936
|
-
this.logger.warn(
|
|
937
|
-
`Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
|
|
1122
|
+
async preparePayload(topicOrDesc, messages) {
|
|
1123
|
+
registerSchema(topicOrDesc, this.schemaRegistry);
|
|
1124
|
+
const payload = await buildSendPayload(
|
|
1125
|
+
topicOrDesc,
|
|
1126
|
+
messages,
|
|
1127
|
+
this.producerOpsDeps
|
|
938
1128
|
);
|
|
1129
|
+
await this.ensureTopic(payload.topic);
|
|
1130
|
+
return payload;
|
|
939
1131
|
}
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
if (this.onRebalance) {
|
|
946
|
-
const onRebalance = this.onRebalance;
|
|
947
|
-
config["rebalance_cb"] = (err, assignment) => {
|
|
948
|
-
const type = err.code === -175 ? "assign" : "revoke";
|
|
949
|
-
try {
|
|
950
|
-
onRebalance(
|
|
951
|
-
type,
|
|
952
|
-
assignment.map((p) => ({
|
|
953
|
-
topic: p.topic,
|
|
954
|
-
partition: p.partition
|
|
955
|
-
}))
|
|
956
|
-
);
|
|
957
|
-
} catch (e) {
|
|
958
|
-
this.logger.warn(
|
|
959
|
-
`onRebalance callback threw: ${e.message}`
|
|
960
|
-
);
|
|
961
|
-
}
|
|
962
|
-
};
|
|
1132
|
+
// afterSend is called once per message — symmetric with beforeSend in buildSendPayload.
|
|
1133
|
+
notifyAfterSend(topic2, count) {
|
|
1134
|
+
for (let i = 0; i < count; i++) {
|
|
1135
|
+
for (const inst of this.instrumentation) {
|
|
1136
|
+
inst.afterSend?.(topic2);
|
|
963
1137
|
}
|
|
964
|
-
this.consumers.set(groupId, this.kafka.consumer(config));
|
|
965
1138
|
}
|
|
966
|
-
return this.consumers.get(groupId);
|
|
967
1139
|
}
|
|
968
1140
|
/**
|
|
969
1141
|
* Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
|
|
@@ -981,13 +1153,6 @@ var KafkaClient = class {
|
|
|
981
1153
|
}, timeoutMs);
|
|
982
1154
|
return promise;
|
|
983
1155
|
}
|
|
984
|
-
resolveTopicName(topicOrDescriptor) {
|
|
985
|
-
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
986
|
-
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
987
|
-
return topicOrDescriptor.__topic;
|
|
988
|
-
}
|
|
989
|
-
return String(topicOrDescriptor);
|
|
990
|
-
}
|
|
991
1156
|
async ensureTopic(topic2) {
|
|
992
1157
|
if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
|
|
993
1158
|
if (!this.isAdminConnected) {
|
|
@@ -999,54 +1164,6 @@ var KafkaClient = class {
|
|
|
999
1164
|
});
|
|
1000
1165
|
this.ensuredTopics.add(topic2);
|
|
1001
1166
|
}
|
|
1002
|
-
/** Register schema from descriptor into global registry (side-effect). */
|
|
1003
|
-
registerSchema(topicOrDesc) {
|
|
1004
|
-
if (topicOrDesc?.__schema) {
|
|
1005
|
-
const topic2 = this.resolveTopicName(topicOrDesc);
|
|
1006
|
-
this.schemaRegistry.set(topic2, topicOrDesc.__schema);
|
|
1007
|
-
}
|
|
1008
|
-
}
|
|
1009
|
-
/** Validate message against schema. Pure — no side-effects on registry. */
|
|
1010
|
-
async validateMessage(topicOrDesc, message) {
|
|
1011
|
-
if (topicOrDesc?.__schema) {
|
|
1012
|
-
return await topicOrDesc.__schema.parse(message);
|
|
1013
|
-
}
|
|
1014
|
-
if (this.strictSchemasEnabled && typeof topicOrDesc === "string") {
|
|
1015
|
-
const schema = this.schemaRegistry.get(topicOrDesc);
|
|
1016
|
-
if (schema) return await schema.parse(message);
|
|
1017
|
-
}
|
|
1018
|
-
return message;
|
|
1019
|
-
}
|
|
1020
|
-
/**
|
|
1021
|
-
* Build a kafkajs-ready send payload.
|
|
1022
|
-
* Handles: topic resolution, schema registration, validation, JSON serialization,
|
|
1023
|
-
* envelope header generation, and instrumentation hooks.
|
|
1024
|
-
*/
|
|
1025
|
-
async buildSendPayload(topicOrDesc, messages) {
|
|
1026
|
-
this.registerSchema(topicOrDesc);
|
|
1027
|
-
const topic2 = this.resolveTopicName(topicOrDesc);
|
|
1028
|
-
const builtMessages = await Promise.all(
|
|
1029
|
-
messages.map(async (m) => {
|
|
1030
|
-
const envelopeHeaders = buildEnvelopeHeaders({
|
|
1031
|
-
correlationId: m.correlationId,
|
|
1032
|
-
schemaVersion: m.schemaVersion,
|
|
1033
|
-
eventId: m.eventId,
|
|
1034
|
-
headers: m.headers
|
|
1035
|
-
});
|
|
1036
|
-
for (const inst of this.instrumentation) {
|
|
1037
|
-
inst.beforeSend?.(topic2, envelopeHeaders);
|
|
1038
|
-
}
|
|
1039
|
-
return {
|
|
1040
|
-
value: JSON.stringify(
|
|
1041
|
-
await this.validateMessage(topicOrDesc, m.value)
|
|
1042
|
-
),
|
|
1043
|
-
key: m.key ?? null,
|
|
1044
|
-
headers: envelopeHeaders
|
|
1045
|
-
};
|
|
1046
|
-
})
|
|
1047
|
-
);
|
|
1048
|
-
return { topic: topic2, messages: builtMessages };
|
|
1049
|
-
}
|
|
1050
1167
|
/** Shared consumer setup: groupId check, schema map, connect, subscribe. */
|
|
1051
1168
|
async setupConsumer(topics, mode, options) {
|
|
1052
1169
|
const {
|
|
@@ -1065,15 +1182,18 @@ var KafkaClient = class {
|
|
|
1065
1182
|
`Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
|
|
1066
1183
|
);
|
|
1067
1184
|
}
|
|
1068
|
-
const consumer =
|
|
1185
|
+
const consumer = getOrCreateConsumer(
|
|
1069
1186
|
gid,
|
|
1070
1187
|
fromBeginning,
|
|
1071
|
-
options.autoCommit ?? true
|
|
1188
|
+
options.autoCommit ?? true,
|
|
1189
|
+
this.consumerOpsDeps
|
|
1072
1190
|
);
|
|
1073
|
-
const schemaMap =
|
|
1074
|
-
|
|
1075
|
-
|
|
1191
|
+
const schemaMap = buildSchemaMap(
|
|
1192
|
+
topics,
|
|
1193
|
+
this.schemaRegistry,
|
|
1194
|
+
optionSchemas
|
|
1076
1195
|
);
|
|
1196
|
+
const topicNames = topics.map((t) => resolveTopicName(t));
|
|
1077
1197
|
for (const t of topicNames) {
|
|
1078
1198
|
await this.ensureTopic(t);
|
|
1079
1199
|
}
|
|
@@ -1094,37 +1214,58 @@ var KafkaClient = class {
|
|
|
1094
1214
|
);
|
|
1095
1215
|
return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry };
|
|
1096
1216
|
}
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1217
|
+
// ── Deps object getters ──────────────────────────────────────────
|
|
1218
|
+
get producerOpsDeps() {
|
|
1219
|
+
return {
|
|
1220
|
+
schemaRegistry: this.schemaRegistry,
|
|
1221
|
+
strictSchemasEnabled: this.strictSchemasEnabled,
|
|
1222
|
+
instrumentation: this.instrumentation
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
1225
|
+
get consumerOpsDeps() {
|
|
1226
|
+
return {
|
|
1227
|
+
consumers: this.consumers,
|
|
1228
|
+
consumerCreationOptions: this.consumerCreationOptions,
|
|
1229
|
+
kafka: this.kafka,
|
|
1230
|
+
onRebalance: this.onRebalance,
|
|
1231
|
+
logger: this.logger
|
|
1232
|
+
};
|
|
1233
|
+
}
|
|
1234
|
+
get messageDeps() {
|
|
1235
|
+
return {
|
|
1236
|
+
logger: this.logger,
|
|
1237
|
+
producer: this.producer,
|
|
1238
|
+
instrumentation: this.instrumentation,
|
|
1239
|
+
onMessageLost: this.onMessageLost
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
get retryTopicDeps() {
|
|
1243
|
+
return {
|
|
1244
|
+
logger: this.logger,
|
|
1245
|
+
producer: this.producer,
|
|
1246
|
+
instrumentation: this.instrumentation,
|
|
1247
|
+
onMessageLost: this.onMessageLost,
|
|
1248
|
+
ensureTopic: (t) => this.ensureTopic(t),
|
|
1249
|
+
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.consumerOpsDeps),
|
|
1250
|
+
runningConsumers: this.runningConsumers
|
|
1251
|
+
};
|
|
1113
1252
|
}
|
|
1114
1253
|
};
|
|
1115
1254
|
|
|
1116
|
-
// src/client/topic.ts
|
|
1255
|
+
// src/client/message/topic.ts
|
|
1117
1256
|
function topic(name) {
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1257
|
+
return {
|
|
1258
|
+
/** Provide an explicit message type without a runtime schema. */
|
|
1259
|
+
type: () => ({
|
|
1260
|
+
__topic: name,
|
|
1261
|
+
__type: void 0
|
|
1262
|
+
}),
|
|
1263
|
+
schema: (schema) => ({
|
|
1264
|
+
__topic: name,
|
|
1265
|
+
__type: void 0,
|
|
1266
|
+
__schema: schema
|
|
1267
|
+
})
|
|
1268
|
+
};
|
|
1128
1269
|
}
|
|
1129
1270
|
|
|
1130
1271
|
// src/nest/kafka.module.ts
|
|
@@ -1243,35 +1384,17 @@ var KafkaModule = class {
|
|
|
1243
1384
|
const token = getKafkaClientToken(options.name);
|
|
1244
1385
|
const kafkaClientProvider = {
|
|
1245
1386
|
provide: token,
|
|
1246
|
-
useFactory:
|
|
1247
|
-
const client = new KafkaClient(
|
|
1248
|
-
options.clientId,
|
|
1249
|
-
options.groupId,
|
|
1250
|
-
options.brokers,
|
|
1251
|
-
{
|
|
1252
|
-
autoCreateTopics: options.autoCreateTopics,
|
|
1253
|
-
strictSchemas: options.strictSchemas,
|
|
1254
|
-
numPartitions: options.numPartitions,
|
|
1255
|
-
instrumentation: options.instrumentation,
|
|
1256
|
-
logger: new import_common3.Logger(`KafkaClient:${options.clientId}`)
|
|
1257
|
-
}
|
|
1258
|
-
);
|
|
1259
|
-
await client.connectProducer();
|
|
1260
|
-
return client;
|
|
1261
|
-
}
|
|
1262
|
-
};
|
|
1263
|
-
const destroyProvider = {
|
|
1264
|
-
provide: `${token}_DESTROY`,
|
|
1265
|
-
useFactory: (client) => ({
|
|
1266
|
-
onModuleDestroy: () => client.disconnect()
|
|
1267
|
-
}),
|
|
1268
|
-
inject: [token]
|
|
1387
|
+
useFactory: () => KafkaModule.buildClient(options)
|
|
1269
1388
|
};
|
|
1270
1389
|
return {
|
|
1271
1390
|
global: options.isGlobal ?? false,
|
|
1272
1391
|
module: KafkaModule,
|
|
1273
1392
|
imports: [import_core2.DiscoveryModule],
|
|
1274
|
-
providers: [
|
|
1393
|
+
providers: [
|
|
1394
|
+
kafkaClientProvider,
|
|
1395
|
+
KafkaModule.buildDestroyProvider(token),
|
|
1396
|
+
KafkaExplorer
|
|
1397
|
+
],
|
|
1275
1398
|
exports: [kafkaClientProvider]
|
|
1276
1399
|
};
|
|
1277
1400
|
}
|
|
@@ -1280,40 +1403,48 @@ var KafkaModule = class {
|
|
|
1280
1403
|
const token = getKafkaClientToken(asyncOptions.name);
|
|
1281
1404
|
const kafkaClientProvider = {
|
|
1282
1405
|
provide: token,
|
|
1283
|
-
useFactory: async (...args) =>
|
|
1284
|
-
const options = await asyncOptions.useFactory(...args);
|
|
1285
|
-
const client = new KafkaClient(
|
|
1286
|
-
options.clientId,
|
|
1287
|
-
options.groupId,
|
|
1288
|
-
options.brokers,
|
|
1289
|
-
{
|
|
1290
|
-
autoCreateTopics: options.autoCreateTopics,
|
|
1291
|
-
strictSchemas: options.strictSchemas,
|
|
1292
|
-
numPartitions: options.numPartitions,
|
|
1293
|
-
instrumentation: options.instrumentation,
|
|
1294
|
-
logger: new import_common3.Logger(`KafkaClient:${options.clientId}`)
|
|
1295
|
-
}
|
|
1296
|
-
);
|
|
1297
|
-
await client.connectProducer();
|
|
1298
|
-
return client;
|
|
1299
|
-
},
|
|
1406
|
+
useFactory: async (...args) => KafkaModule.buildClient(await asyncOptions.useFactory(...args)),
|
|
1300
1407
|
inject: asyncOptions.inject || []
|
|
1301
1408
|
};
|
|
1302
|
-
const destroyProvider = {
|
|
1303
|
-
provide: `${token}_DESTROY`,
|
|
1304
|
-
useFactory: (client) => ({
|
|
1305
|
-
onModuleDestroy: () => client.disconnect()
|
|
1306
|
-
}),
|
|
1307
|
-
inject: [token]
|
|
1308
|
-
};
|
|
1309
1409
|
return {
|
|
1310
1410
|
global: asyncOptions.isGlobal ?? false,
|
|
1311
1411
|
module: KafkaModule,
|
|
1312
1412
|
imports: [...asyncOptions.imports || [], import_core2.DiscoveryModule],
|
|
1313
|
-
providers: [
|
|
1413
|
+
providers: [
|
|
1414
|
+
kafkaClientProvider,
|
|
1415
|
+
KafkaModule.buildDestroyProvider(token),
|
|
1416
|
+
KafkaExplorer
|
|
1417
|
+
],
|
|
1314
1418
|
exports: [kafkaClientProvider]
|
|
1315
1419
|
};
|
|
1316
1420
|
}
|
|
1421
|
+
static async buildClient(options) {
|
|
1422
|
+
const client = new KafkaClient(
|
|
1423
|
+
options.clientId,
|
|
1424
|
+
options.groupId,
|
|
1425
|
+
options.brokers,
|
|
1426
|
+
{
|
|
1427
|
+
autoCreateTopics: options.autoCreateTopics,
|
|
1428
|
+
strictSchemas: options.strictSchemas,
|
|
1429
|
+
numPartitions: options.numPartitions,
|
|
1430
|
+
instrumentation: options.instrumentation,
|
|
1431
|
+
onMessageLost: options.onMessageLost,
|
|
1432
|
+
onRebalance: options.onRebalance,
|
|
1433
|
+
logger: new import_common3.Logger(`KafkaClient:${options.clientId}`)
|
|
1434
|
+
}
|
|
1435
|
+
);
|
|
1436
|
+
await client.connectProducer();
|
|
1437
|
+
return client;
|
|
1438
|
+
}
|
|
1439
|
+
static buildDestroyProvider(token) {
|
|
1440
|
+
return {
|
|
1441
|
+
provide: `${token}_DESTROY`,
|
|
1442
|
+
useFactory: (client) => ({
|
|
1443
|
+
onModuleDestroy: () => client.disconnect()
|
|
1444
|
+
}),
|
|
1445
|
+
inject: [token]
|
|
1446
|
+
};
|
|
1447
|
+
}
|
|
1317
1448
|
};
|
|
1318
1449
|
KafkaModule = __decorateClass([
|
|
1319
1450
|
(0, import_common3.Module)({})
|
|
@@ -1323,15 +1454,7 @@ KafkaModule = __decorateClass([
|
|
|
1323
1454
|
var import_common4 = require("@nestjs/common");
|
|
1324
1455
|
var KafkaHealthIndicator = class {
|
|
1325
1456
|
async check(client) {
|
|
1326
|
-
|
|
1327
|
-
return await client.checkStatus();
|
|
1328
|
-
} catch (error) {
|
|
1329
|
-
return {
|
|
1330
|
-
status: "down",
|
|
1331
|
-
clientId: client.clientId,
|
|
1332
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1333
|
-
};
|
|
1334
|
-
}
|
|
1457
|
+
return client.checkStatus();
|
|
1335
1458
|
}
|
|
1336
1459
|
};
|
|
1337
1460
|
KafkaHealthIndicator = __decorateClass([
|