@relaybus/kafka 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +469 -415
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -4
- package/dist/index.d.ts +5 -4
- package/dist/index.js +470 -416
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -312,25 +312,17 @@ function plainJsonDecoder(options) {
|
|
|
312
312
|
if (record.headers?.[SPEC_VERSION_HEADER] !== void 0 || internal.mediaTypeOf(headers[CONTENT_TYPE]) === STRUCTURED_MEDIA_TYPE2) {
|
|
313
313
|
return decodeCloudEvent(record);
|
|
314
314
|
}
|
|
315
|
-
const plain =
|
|
316
|
-
|
|
317
|
-
partition: record.partition,
|
|
318
|
-
offset: record.offset,
|
|
319
|
-
key: readKeyLeniently(record.key),
|
|
320
|
-
headers,
|
|
321
|
-
payload: parsePayload(record.value),
|
|
322
|
-
timestamp: record.timestamp
|
|
323
|
-
};
|
|
324
|
-
const type = resolveText(options.type, { record: plain, name: "type" });
|
|
315
|
+
const plain = plainRecordOf(record, headers);
|
|
316
|
+
const type = KAFKA_EVENTS.text(options.type, { plain, name: "type" });
|
|
325
317
|
if (type === void 0) {
|
|
326
318
|
throw invalid("Record has no event type: the type option found none");
|
|
327
319
|
}
|
|
328
320
|
assertValidType(type, "The event type");
|
|
329
321
|
return {
|
|
330
|
-
id:
|
|
322
|
+
id: KAFKA_EVENTS.text(options.id, { plain, name: "id" }) ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
|
|
331
323
|
type,
|
|
332
|
-
source:
|
|
333
|
-
key: options.key === void 0 ? plain.key :
|
|
324
|
+
source: KAFKA_EVENTS.text(options.source, { plain, name: "source" }) ?? record.topic,
|
|
325
|
+
key: options.key === void 0 ? plain.key : KAFKA_EVENTS.text(options.key, { plain, name: "key" }) ?? null,
|
|
334
326
|
payload: plain.payload,
|
|
335
327
|
headers,
|
|
336
328
|
correlationId: null,
|
|
@@ -339,27 +331,38 @@ function plainJsonDecoder(options) {
|
|
|
339
331
|
};
|
|
340
332
|
};
|
|
341
333
|
}
|
|
342
|
-
function
|
|
343
|
-
if (typeof decoded !== "object" || decoded === null) {
|
|
344
|
-
throw invalid("The decoder must return an event object synchronously");
|
|
345
|
-
}
|
|
346
|
-
if ("then" in decoded) {
|
|
347
|
-
observeRejection(decoded);
|
|
348
|
-
throw invalid("The decoder must return an event object synchronously");
|
|
349
|
-
}
|
|
350
|
-
const event = decoded;
|
|
334
|
+
function plainRecordOf(record, headers) {
|
|
351
335
|
return {
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
key:
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
causationId: event.causationId ?? null,
|
|
360
|
-
occurredAt: event.occurredAt ?? requireRecordTimestamp(record)
|
|
336
|
+
topic: record.topic,
|
|
337
|
+
partition: record.partition,
|
|
338
|
+
offset: record.offset,
|
|
339
|
+
key: readKeyLeniently(record.key),
|
|
340
|
+
headers,
|
|
341
|
+
payload: parsePayload(record.value),
|
|
342
|
+
timestamp: record.timestamp
|
|
361
343
|
};
|
|
362
344
|
}
|
|
345
|
+
var KafkaDecodedEvents = class extends internal.DecodedEventReader {
|
|
346
|
+
invalid(reason) {
|
|
347
|
+
return invalid(reason);
|
|
348
|
+
}
|
|
349
|
+
/** The topic. */
|
|
350
|
+
defaultSource(record) {
|
|
351
|
+
return record.topic;
|
|
352
|
+
}
|
|
353
|
+
/** The record key as UTF-8 text. */
|
|
354
|
+
defaultKey(record) {
|
|
355
|
+
return readRecordKey(record.key);
|
|
356
|
+
}
|
|
357
|
+
/** The record timestamp; a record without a usable one is malformed. */
|
|
358
|
+
defaultTime(record) {
|
|
359
|
+
return requireRecordTimestamp(record);
|
|
360
|
+
}
|
|
361
|
+
};
|
|
362
|
+
var KAFKA_EVENTS = new KafkaDecodedEvents();
|
|
363
|
+
function completeDecodedEvent(decoded, record) {
|
|
364
|
+
return KAFKA_EVENTS.complete(decoded, record);
|
|
365
|
+
}
|
|
363
366
|
function extensionAttributes(body) {
|
|
364
367
|
return Object.fromEntries(
|
|
365
368
|
Object.entries(body).filter(
|
|
@@ -405,33 +408,17 @@ function readKeyLeniently(key) {
|
|
|
405
408
|
return Buffer.from(key).toString("base64");
|
|
406
409
|
}
|
|
407
410
|
}
|
|
408
|
-
function resolveText(option, { record, name }) {
|
|
409
|
-
const value = typeof option === "function" ? option(record) : option;
|
|
410
|
-
if (value === void 0 || value === null) {
|
|
411
|
-
return void 0;
|
|
412
|
-
}
|
|
413
|
-
if (typeof value !== "string") {
|
|
414
|
-
observeRejection(value);
|
|
415
|
-
throw invalid(`The ${name} option returned ${typeof value}, expected a string`);
|
|
416
|
-
}
|
|
417
|
-
return value.length === 0 ? void 0 : value;
|
|
418
|
-
}
|
|
419
411
|
function resolveOccurredAt(option, { plain, record }) {
|
|
420
412
|
const value = option?.(plain);
|
|
421
413
|
if (value === void 0) {
|
|
422
414
|
return requireRecordTimestamp(record);
|
|
423
415
|
}
|
|
424
416
|
if (!(value instanceof Date) || Number.isNaN(value.getTime())) {
|
|
425
|
-
observeRejection(value);
|
|
417
|
+
internal.observeRejection(value);
|
|
426
418
|
throw invalid("The occurredAt option returned no valid Date");
|
|
427
419
|
}
|
|
428
420
|
return value;
|
|
429
421
|
}
|
|
430
|
-
function observeRejection(value) {
|
|
431
|
-
if (typeof value === "object" && value !== null && "then" in value) {
|
|
432
|
-
Promise.resolve(value).catch(() => void 0);
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
422
|
|
|
436
423
|
// src/codecs.ts
|
|
437
424
|
var cloudEventsBinary = Object.freeze({
|
|
@@ -454,32 +441,34 @@ function jsonEnvelope(mapping) {
|
|
|
454
441
|
const stampsTime = mapping.occurredAt === void 0;
|
|
455
442
|
const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { [CONTENT_TYPE]: contentType, [EVENT_ID_HEADER]: envelope.id } : { [CONTENT_TYPE]: contentType };
|
|
456
443
|
return {
|
|
457
|
-
encode: (envelope) => {
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
444
|
+
encode: (envelope) => encodeJson(envelope, { format, stampsTime, ownHeaders }),
|
|
445
|
+
decode: (record) => decodeJson(record, format)
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
function encodeJson(envelope, { format, stampsTime, ownHeaders }) {
|
|
449
|
+
assertRepresentable(envelope);
|
|
450
|
+
const { body, headers } = format.write(envelope);
|
|
451
|
+
const taken = [CONTENT_TYPE, EVENT_ID_HEADER].find((name) => Object.hasOwn(headers, name));
|
|
452
|
+
if (taken !== void 0) {
|
|
453
|
+
throw new UnencodableEventError(
|
|
454
|
+
`Event ${envelope.id} has a header '${taken}', which the codec writes itself`
|
|
455
|
+
);
|
|
456
|
+
}
|
|
457
|
+
return {
|
|
458
|
+
key: envelope.key,
|
|
459
|
+
value: serializeJson(envelope, body),
|
|
460
|
+
headers: { ...headers, ...ownHeaders(envelope) },
|
|
461
|
+
...stampsTime ? { timestamp: recordTimestamp(envelope) } : {}
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
function decodeJson(record, format) {
|
|
465
|
+
const { [EVENT_ID_HEADER]: eventId, ...headers } = withoutContentType(
|
|
466
|
+
readHeadersLeniently(record.headers)
|
|
467
|
+
);
|
|
468
|
+
const fields = format.read({ body: parsePayload(record.value), headers });
|
|
469
|
+
return {
|
|
470
|
+
...fields,
|
|
471
|
+
id: fields.id ?? (eventId === void 0 || eventId === "" ? `kafka:${record.topic}:${record.partition}:${record.offset}` : eventId)
|
|
483
472
|
};
|
|
484
473
|
}
|
|
485
474
|
function withoutContentType(headers) {
|
|
@@ -497,6 +486,139 @@ function recordTimestamp(envelope) {
|
|
|
497
486
|
formatTime(envelope);
|
|
498
487
|
return String(envelope.occurredAt.getTime());
|
|
499
488
|
}
|
|
489
|
+
function tracksMembership(consumer) {
|
|
490
|
+
return consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0;
|
|
491
|
+
}
|
|
492
|
+
var GroupJoinWatch = class {
|
|
493
|
+
#settings;
|
|
494
|
+
#waiter;
|
|
495
|
+
constructor(settings) {
|
|
496
|
+
this.#settings = settings;
|
|
497
|
+
}
|
|
498
|
+
/** The consumer joined its group. */
|
|
499
|
+
joined() {
|
|
500
|
+
this.#waiter?.resolve();
|
|
501
|
+
}
|
|
502
|
+
/** The consumer crashed while it was joining. */
|
|
503
|
+
failed(error) {
|
|
504
|
+
this.#waiter?.reject(error);
|
|
505
|
+
}
|
|
506
|
+
/** The transport closed while the consumer was joining. */
|
|
507
|
+
closed() {
|
|
508
|
+
this.#waiter?.reject(new TransportClosedError(`Transport '${this.#settings.name}' is closed`));
|
|
509
|
+
}
|
|
510
|
+
/** Connects and subscribes the consumer, runs it and resolves once it joined its group. */
|
|
511
|
+
async start(consumer, {
|
|
512
|
+
topics,
|
|
513
|
+
fromBeginning,
|
|
514
|
+
eachMessage
|
|
515
|
+
}) {
|
|
516
|
+
await consumer.connect();
|
|
517
|
+
await consumer.subscribe({ topics, fromBeginning });
|
|
518
|
+
const joined = this.#wait(consumer);
|
|
519
|
+
try {
|
|
520
|
+
await consumer.run({ eachMessage });
|
|
521
|
+
await joined.promise;
|
|
522
|
+
} finally {
|
|
523
|
+
joined.dispose();
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
#wait(consumer) {
|
|
527
|
+
if (!tracksMembership(consumer)) {
|
|
528
|
+
return { promise: Promise.resolve(), dispose: () => void 0 };
|
|
529
|
+
}
|
|
530
|
+
const { name, groupId, timings } = this.#settings;
|
|
531
|
+
let cancel;
|
|
532
|
+
const promise = new Promise((resolve, reject) => {
|
|
533
|
+
this.#waiter = { resolve, reject };
|
|
534
|
+
cancel = internal.schedule(core.systemClock, {
|
|
535
|
+
run: () => {
|
|
536
|
+
reject(
|
|
537
|
+
new GroupJoinTimeoutError(
|
|
538
|
+
`Consumer of transport '${name}' did not join group '${groupId}' within ${timings.startupTimeoutMs}ms`
|
|
539
|
+
)
|
|
540
|
+
);
|
|
541
|
+
},
|
|
542
|
+
delayMs: timings.startupTimeoutMs
|
|
543
|
+
});
|
|
544
|
+
});
|
|
545
|
+
promise.catch(() => void 0);
|
|
546
|
+
return {
|
|
547
|
+
promise,
|
|
548
|
+
dispose: () => {
|
|
549
|
+
cancel?.();
|
|
550
|
+
this.#waiter = void 0;
|
|
551
|
+
}
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
};
|
|
555
|
+
var KafkaProducerSession = class {
|
|
556
|
+
#settings;
|
|
557
|
+
/** Whether the transport is closed: then nothing connects any more. */
|
|
558
|
+
#isClosed;
|
|
559
|
+
#producer;
|
|
560
|
+
#connected = false;
|
|
561
|
+
constructor(settings, isClosed) {
|
|
562
|
+
this.#settings = settings;
|
|
563
|
+
this.#isClosed = isClosed;
|
|
564
|
+
}
|
|
565
|
+
/** True once the producer connected, until the transport closes. */
|
|
566
|
+
get isConnected() {
|
|
567
|
+
return this.#connected;
|
|
568
|
+
}
|
|
569
|
+
/** The producer opened so far, if any: what closing must disconnect. */
|
|
570
|
+
get pending() {
|
|
571
|
+
return this.#producer;
|
|
572
|
+
}
|
|
573
|
+
markClosed() {
|
|
574
|
+
this.#connected = false;
|
|
575
|
+
}
|
|
576
|
+
connected() {
|
|
577
|
+
if (this.#isClosed()) {
|
|
578
|
+
return Promise.reject(
|
|
579
|
+
new TransportClosedError(`Transport '${this.#settings.name}' is closed`)
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
this.#producer ??= this.#open();
|
|
583
|
+
return this.#producer;
|
|
584
|
+
}
|
|
585
|
+
/** Sends with the publish timeout; the late outcome of a timed-out send is ignored. */
|
|
586
|
+
async send(producer, record) {
|
|
587
|
+
const { name, timings } = this.#settings;
|
|
588
|
+
let cancel;
|
|
589
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
590
|
+
cancel = internal.schedule(core.systemClock, {
|
|
591
|
+
run: () => {
|
|
592
|
+
reject(
|
|
593
|
+
new PublishTimeoutError(
|
|
594
|
+
`Transport '${name}' got no broker ack for topic '${record.topic}' within ${timings.publishTimeoutMs}ms`
|
|
595
|
+
)
|
|
596
|
+
);
|
|
597
|
+
},
|
|
598
|
+
delayMs: timings.publishTimeoutMs
|
|
599
|
+
});
|
|
600
|
+
});
|
|
601
|
+
const sending = producer.send(record);
|
|
602
|
+
sending.catch(() => void 0);
|
|
603
|
+
try {
|
|
604
|
+
await Promise.race([sending, timeout]);
|
|
605
|
+
} finally {
|
|
606
|
+
cancel?.();
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
async #open() {
|
|
610
|
+
const { kafka, producerConfig, maxInFlightRequests } = this.#settings;
|
|
611
|
+
const producer = kafka.producer({ ...producerConfig, idempotent: true, maxInFlightRequests });
|
|
612
|
+
try {
|
|
613
|
+
await producer.connect();
|
|
614
|
+
} catch (error) {
|
|
615
|
+
this.#producer = void 0;
|
|
616
|
+
throw error;
|
|
617
|
+
}
|
|
618
|
+
this.#connected = !this.#isClosed();
|
|
619
|
+
return producer;
|
|
620
|
+
}
|
|
621
|
+
};
|
|
500
622
|
var DEFAULTS = {
|
|
501
623
|
name: "kafka",
|
|
502
624
|
maxInFlightRequests: 1,
|
|
@@ -509,44 +631,224 @@ var DEFAULTS = {
|
|
|
509
631
|
restartInitialBackoff: "1s",
|
|
510
632
|
restartMaxBackoff: "30s"
|
|
511
633
|
};
|
|
512
|
-
var
|
|
634
|
+
var MAXIMUM_IN_FLIGHT_REQUESTS = 5;
|
|
635
|
+
function resolveKafkaSettings(options) {
|
|
636
|
+
return {
|
|
637
|
+
name: options.name ?? DEFAULTS.name,
|
|
638
|
+
kafka: options.kafka,
|
|
639
|
+
groupId: options.groupId,
|
|
640
|
+
fromBeginning: options.fromBeginning ?? true,
|
|
641
|
+
maxInFlightRequests: internal.BoundedInteger.of(
|
|
642
|
+
options.maxInFlightRequests ?? DEFAULTS.maxInFlightRequests,
|
|
643
|
+
{ name: "maxInFlightRequests", minimum: 1, maximum: MAXIMUM_IN_FLIGHT_REQUESTS }
|
|
644
|
+
),
|
|
645
|
+
producerConfig: options.producerConfig ?? {},
|
|
646
|
+
consumerConfig: options.consumerConfig ?? {},
|
|
647
|
+
invalidPolicy: options.onInvalidMessage ?? "dead-letter",
|
|
648
|
+
codec: resolveCodec(options),
|
|
649
|
+
logger: options.logger ?? console,
|
|
650
|
+
timings: resolveTimings(options)
|
|
651
|
+
};
|
|
652
|
+
}
|
|
653
|
+
function resolveCodec({ codec, decode }) {
|
|
654
|
+
if (decode === void 0) {
|
|
655
|
+
return codec ?? cloudEventsBinary;
|
|
656
|
+
}
|
|
657
|
+
if (codec !== void 0) {
|
|
658
|
+
throw new core.ConfigurationError("Give 'codec' or the deprecated 'decode', not both");
|
|
659
|
+
}
|
|
660
|
+
return { encode: encodeEnvelope, decode };
|
|
661
|
+
}
|
|
662
|
+
function resolveTimings(options) {
|
|
663
|
+
const processing = options.processing ?? {};
|
|
664
|
+
const maxAttempts = internal.PositiveInteger.of(
|
|
665
|
+
processing.maxAttempts ?? DEFAULTS.maxAttempts,
|
|
666
|
+
"processing.maxAttempts"
|
|
667
|
+
);
|
|
668
|
+
return {
|
|
669
|
+
maxAttempts,
|
|
670
|
+
processingBackoff: new internal.ExponentialBackoff({
|
|
671
|
+
initialMs: internal.Milliseconds.of(
|
|
672
|
+
processing.initialBackoff ?? DEFAULTS.initialBackoff,
|
|
673
|
+
"processing.initialBackoff"
|
|
674
|
+
),
|
|
675
|
+
maxMs: internal.Milliseconds.of(processing.maxBackoff ?? DEFAULTS.maxBackoff, "processing.maxBackoff")
|
|
676
|
+
}),
|
|
677
|
+
startupTimeoutMs: internal.Milliseconds.of(
|
|
678
|
+
options.startupTimeout ?? DEFAULTS.startupTimeout,
|
|
679
|
+
"startupTimeout"
|
|
680
|
+
),
|
|
681
|
+
publishTimeoutMs: internal.Milliseconds.of(
|
|
682
|
+
options.publishTimeout ?? DEFAULTS.publishTimeout,
|
|
683
|
+
"publishTimeout"
|
|
684
|
+
),
|
|
685
|
+
heartbeatIntervalMs: Math.max(
|
|
686
|
+
1,
|
|
687
|
+
internal.Milliseconds.of(
|
|
688
|
+
processing.heartbeatInterval ?? DEFAULTS.heartbeatInterval,
|
|
689
|
+
"processing.heartbeatInterval"
|
|
690
|
+
)
|
|
691
|
+
),
|
|
692
|
+
restartBackoff: resolveRestartBackoff(options.restart ?? {})
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
function resolveRestartBackoff(restart) {
|
|
696
|
+
return new internal.ExponentialBackoff({
|
|
697
|
+
initialMs: internal.Milliseconds.of(
|
|
698
|
+
restart.initialBackoff ?? DEFAULTS.restartInitialBackoff,
|
|
699
|
+
"restart.initialBackoff"
|
|
700
|
+
),
|
|
701
|
+
maxMs: internal.Milliseconds.of(restart.maxBackoff ?? DEFAULTS.restartMaxBackoff, "restart.maxBackoff")
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
function deadLetterTopicFor(policy, sourceTopic) {
|
|
705
|
+
if (policy === "dead-letter") {
|
|
706
|
+
return `${sourceTopic}.dead`;
|
|
707
|
+
}
|
|
708
|
+
if (typeof policy === "object") {
|
|
709
|
+
const { deadLetterTopic } = policy;
|
|
710
|
+
return typeof deadLetterTopic === "function" ? deadLetterTopic(sourceTopic) : deadLetterTopic;
|
|
711
|
+
}
|
|
712
|
+
return void 0;
|
|
713
|
+
}
|
|
714
|
+
function toHandlerFailure(error) {
|
|
715
|
+
const { name, message } = core.toErrorInfo(error);
|
|
716
|
+
return new HandlerFailedError(`${name}: ${message}`);
|
|
717
|
+
}
|
|
718
|
+
function assertCanSubscribe(state, {
|
|
719
|
+
settings,
|
|
720
|
+
destinations
|
|
721
|
+
}) {
|
|
722
|
+
const { name, groupId, invalidPolicy } = settings;
|
|
723
|
+
if (state === "closed") {
|
|
724
|
+
throw new TransportClosedError(`Transport '${name}' is closed`);
|
|
725
|
+
}
|
|
726
|
+
if (state === "subscribed") {
|
|
727
|
+
throw new core.ConfigurationError(
|
|
728
|
+
`Transport '${name}' already subscribed; list every topic in one subscription or use one KafkaTransport per consumer group`
|
|
729
|
+
);
|
|
730
|
+
}
|
|
731
|
+
if (groupId === void 0 || groupId.length === 0) {
|
|
732
|
+
throw new core.ConfigurationError(`Transport '${name}' needs a groupId to subscribe`);
|
|
733
|
+
}
|
|
734
|
+
if (destinations.length === 0) {
|
|
735
|
+
throw new core.ConfigurationError(`Transport '${name}' subscribed to no topics`);
|
|
736
|
+
}
|
|
737
|
+
for (const topic of destinations) {
|
|
738
|
+
const deadLetterTopic = deadLetterTopicFor(invalidPolicy, topic);
|
|
739
|
+
if (deadLetterTopic !== void 0 && destinations.includes(deadLetterTopic)) {
|
|
740
|
+
throw new core.ConfigurationError(
|
|
741
|
+
`Dead letter topic '${deadLetterTopic}' is also consumed, which would loop`
|
|
742
|
+
);
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
// src/kafka-records.ts
|
|
748
|
+
function decodeRecord(codec, { topic, partition, message }) {
|
|
749
|
+
if (codec === cloudEventsBinary) {
|
|
750
|
+
return decodeEnvelope(message);
|
|
751
|
+
}
|
|
752
|
+
const record = {
|
|
753
|
+
topic,
|
|
754
|
+
partition,
|
|
755
|
+
offset: message.offset,
|
|
756
|
+
key: message.key,
|
|
757
|
+
value: message.value,
|
|
758
|
+
headers: message.headers,
|
|
759
|
+
timestamp: message.timestamp
|
|
760
|
+
};
|
|
761
|
+
return completeDecodedEvent(codec.decode(record), record);
|
|
762
|
+
}
|
|
763
|
+
function deadLetterRecord({ topic, partition, message }, {
|
|
764
|
+
reason,
|
|
765
|
+
name,
|
|
766
|
+
code,
|
|
767
|
+
deadLetterTopic
|
|
768
|
+
}) {
|
|
769
|
+
const headers = {
|
|
770
|
+
...toOutgoingHeaders(message.headers),
|
|
771
|
+
"relaybus-invalid-reason": reason,
|
|
772
|
+
"relaybus-source-topic": topic,
|
|
773
|
+
"relaybus-source-partition": String(partition),
|
|
774
|
+
"relaybus-source-offset": message.offset,
|
|
775
|
+
"relaybus-error-name": name
|
|
776
|
+
};
|
|
777
|
+
if (code === void 0) {
|
|
778
|
+
delete headers["relaybus-error-code"];
|
|
779
|
+
} else {
|
|
780
|
+
headers["relaybus-error-code"] = code;
|
|
781
|
+
}
|
|
782
|
+
return {
|
|
783
|
+
topic: deadLetterTopic,
|
|
784
|
+
acks: -1,
|
|
785
|
+
messages: [
|
|
786
|
+
{
|
|
787
|
+
key: message.key === null ? null : Buffer.from(message.key),
|
|
788
|
+
value: message.value === null ? null : Buffer.from(message.value),
|
|
789
|
+
headers
|
|
790
|
+
}
|
|
791
|
+
]
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
var WakeableSleeps = class {
|
|
795
|
+
#sleepers = /* @__PURE__ */ new Set();
|
|
796
|
+
sleep(milliseconds) {
|
|
797
|
+
return new Promise((resolve) => {
|
|
798
|
+
const wake = () => {
|
|
799
|
+
cancel();
|
|
800
|
+
this.#sleepers.delete(wake);
|
|
801
|
+
resolve();
|
|
802
|
+
};
|
|
803
|
+
const cancel = internal.schedule(core.systemClock, { run: wake, delayMs: milliseconds });
|
|
804
|
+
this.#sleepers.add(wake);
|
|
805
|
+
});
|
|
806
|
+
}
|
|
807
|
+
wakeAll() {
|
|
808
|
+
for (const wake of this.#sleepers) {
|
|
809
|
+
wake();
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
/**
|
|
813
|
+
* Sleeps `milliseconds` in slices of at most `sliceMs`, running `between` after each, until done
|
|
814
|
+
* or `stopped`: a long backoff then never outlives the session timeout.
|
|
815
|
+
*/
|
|
816
|
+
async sleepInSlices(milliseconds, {
|
|
817
|
+
sliceMs,
|
|
818
|
+
between,
|
|
819
|
+
stopped
|
|
820
|
+
}) {
|
|
821
|
+
let remaining = milliseconds;
|
|
822
|
+
do {
|
|
823
|
+
const slice = Math.min(remaining, sliceMs);
|
|
824
|
+
await this.sleep(slice);
|
|
825
|
+
await between();
|
|
826
|
+
remaining -= slice;
|
|
827
|
+
} while (remaining > 0 && !stopped());
|
|
828
|
+
}
|
|
829
|
+
};
|
|
830
|
+
|
|
831
|
+
// src/kafka-transport.ts
|
|
832
|
+
var KafkaTransport = class extends internal.SubscribingTransport {
|
|
513
833
|
name;
|
|
514
834
|
system = "kafka";
|
|
515
|
-
#kafka;
|
|
516
|
-
#groupId;
|
|
517
|
-
#fromBeginning;
|
|
518
|
-
#maxInFlightRequests;
|
|
519
|
-
#producerConfig;
|
|
520
|
-
#consumerConfig;
|
|
521
|
-
#invalidPolicy;
|
|
522
|
-
#codec;
|
|
523
|
-
#logger;
|
|
524
835
|
#settings;
|
|
525
|
-
#sleepers = /* @__PURE__ */ new Set();
|
|
526
|
-
#state = "idle";
|
|
527
836
|
#producer;
|
|
837
|
+
#groupJoin;
|
|
838
|
+
#sleeps = new WakeableSleeps();
|
|
839
|
+
#state = "idle";
|
|
528
840
|
#consumer;
|
|
529
841
|
#groupJoined = false;
|
|
530
842
|
#consuming = false;
|
|
531
|
-
#producerConnected = false;
|
|
532
843
|
#restartLoop;
|
|
533
844
|
#stalledPartitions = /* @__PURE__ */ new Set();
|
|
534
845
|
#startConsuming;
|
|
535
|
-
#joinWaiter;
|
|
536
846
|
constructor(options) {
|
|
537
|
-
|
|
538
|
-
this.#
|
|
539
|
-
this
|
|
540
|
-
this.#
|
|
541
|
-
this.#
|
|
542
|
-
options.maxInFlightRequests ?? DEFAULTS.maxInFlightRequests
|
|
543
|
-
);
|
|
544
|
-
this.#producerConfig = options.producerConfig ?? {};
|
|
545
|
-
this.#consumerConfig = options.consumerConfig ?? {};
|
|
546
|
-
this.#invalidPolicy = options.onInvalidMessage ?? "dead-letter";
|
|
547
|
-
this.#codec = resolveCodec(options);
|
|
548
|
-
this.#logger = options.logger ?? console;
|
|
549
|
-
this.#settings = resolveSettings(options);
|
|
847
|
+
super();
|
|
848
|
+
this.#settings = resolveKafkaSettings(options);
|
|
849
|
+
this.name = this.#settings.name;
|
|
850
|
+
this.#producer = new KafkaProducerSession(this.#settings, () => this.#state === "closed");
|
|
851
|
+
this.#groupJoin = new GroupJoinWatch(this.#settings);
|
|
550
852
|
}
|
|
551
853
|
/**
|
|
552
854
|
* True while the consumer is joined to its group, not between a crash or
|
|
@@ -563,21 +865,21 @@ var KafkaTransport = class {
|
|
|
563
865
|
* dead consumer is being restarted.
|
|
564
866
|
*/
|
|
565
867
|
isReady() {
|
|
566
|
-
if (this.#state === "closed" || !this.#
|
|
868
|
+
if (this.#state === "closed" || !this.#producer.isConnected) {
|
|
567
869
|
return false;
|
|
568
870
|
}
|
|
569
871
|
return this.#state === "idle" || this.isConsuming();
|
|
570
872
|
}
|
|
571
873
|
/** Connects the producer. Called by the bus on start; `publish` also connects on demand. */
|
|
572
874
|
async connect() {
|
|
573
|
-
await this.#
|
|
875
|
+
await this.#producer.connected();
|
|
574
876
|
}
|
|
575
877
|
async publish(envelope, destination) {
|
|
576
|
-
const producer = await this.#
|
|
577
|
-
await this.#send(producer, {
|
|
878
|
+
const producer = await this.#producer.connected();
|
|
879
|
+
await this.#producer.send(producer, {
|
|
578
880
|
topic: destination,
|
|
579
881
|
acks: -1,
|
|
580
|
-
messages: [this.#codec.encode(envelope)]
|
|
882
|
+
messages: [this.#settings.codec.encode(envelope)]
|
|
581
883
|
});
|
|
582
884
|
}
|
|
583
885
|
/**
|
|
@@ -587,14 +889,21 @@ var KafkaTransport = class {
|
|
|
587
889
|
* @throws ConfigurationError when `groupId` is missing, `destinations` is empty
|
|
588
890
|
* or the transport already subscribed (create one transport per subscription).
|
|
589
891
|
*/
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
892
|
+
openSubscription(subscription) {
|
|
893
|
+
return this.#subscribe(subscription);
|
|
894
|
+
}
|
|
895
|
+
async #subscribe({ destinations, onMessage, hooks }) {
|
|
896
|
+
assertCanSubscribe(this.#state, { settings: this.#settings, destinations });
|
|
593
897
|
this.#state = "subscribed";
|
|
594
|
-
const
|
|
898
|
+
const { kafka, consumerConfig, groupId } = this.#settings;
|
|
899
|
+
const consumer = kafka.consumer({ ...consumerConfig, groupId });
|
|
595
900
|
this.#consumer = consumer;
|
|
596
901
|
this.#observeConsumer(consumer);
|
|
597
|
-
const start = () => this.#
|
|
902
|
+
const start = () => this.#groupJoin.start(consumer, {
|
|
903
|
+
topics: [...destinations],
|
|
904
|
+
fromBeginning: this.#settings.fromBeginning,
|
|
905
|
+
eachMessage: (payload) => this.#handleMessage(payload, { onMessage, hooks })
|
|
906
|
+
});
|
|
598
907
|
this.#startConsuming = start;
|
|
599
908
|
try {
|
|
600
909
|
await start();
|
|
@@ -606,40 +915,21 @@ var KafkaTransport = class {
|
|
|
606
915
|
throw error;
|
|
607
916
|
}
|
|
608
917
|
this.#consuming = true;
|
|
609
|
-
this.#logger.log(
|
|
610
|
-
`[KafkaTransport](subscribe) Consuming | transport=${this.name} group=${
|
|
918
|
+
this.#settings.logger.log(
|
|
919
|
+
`[KafkaTransport](subscribe) Consuming | transport=${this.name} group=${groupId} topics=${destinations.join(",")}`
|
|
611
920
|
);
|
|
612
921
|
}
|
|
613
|
-
async #startConsumer(consumer, {
|
|
614
|
-
destinations,
|
|
615
|
-
onMessage,
|
|
616
|
-
hooks
|
|
617
|
-
}) {
|
|
618
|
-
await consumer.connect();
|
|
619
|
-
await consumer.subscribe({ topics: destinations, fromBeginning: this.#fromBeginning });
|
|
620
|
-
const joined = this.#waitForGroupJoin(consumer);
|
|
621
|
-
try {
|
|
622
|
-
await consumer.run({
|
|
623
|
-
eachMessage: (payload) => this.#handleMessage(payload, { onMessage, hooks })
|
|
624
|
-
});
|
|
625
|
-
await joined.promise;
|
|
626
|
-
} finally {
|
|
627
|
-
joined.dispose();
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
922
|
/** Stops consuming and disconnects the consumer and the producer. Safe to call twice. */
|
|
631
923
|
async close() {
|
|
632
924
|
if (this.#state === "closed") {
|
|
633
925
|
return;
|
|
634
926
|
}
|
|
635
927
|
this.#state = "closed";
|
|
636
|
-
const pendingProducer = this.#producer;
|
|
928
|
+
const pendingProducer = this.#producer.pending;
|
|
637
929
|
this.#groupJoined = false;
|
|
638
|
-
this.#
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
}
|
|
642
|
-
this.#joinWaiter?.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
|
|
930
|
+
this.#producer.markClosed();
|
|
931
|
+
this.#sleeps.wakeAll();
|
|
932
|
+
this.#groupJoin.closed();
|
|
643
933
|
await this.#restartLoop;
|
|
644
934
|
const producer = await pendingProducer?.catch(() => void 0);
|
|
645
935
|
const results = await Promise.allSettled([
|
|
@@ -651,99 +941,15 @@ var KafkaTransport = class {
|
|
|
651
941
|
throw failure.reason;
|
|
652
942
|
}
|
|
653
943
|
}
|
|
654
|
-
#
|
|
655
|
-
if (this.#state === "closed") {
|
|
656
|
-
throw new TransportClosedError(`Transport '${this.name}' is closed`);
|
|
657
|
-
}
|
|
658
|
-
if (this.#state === "subscribed") {
|
|
659
|
-
throw new core.ConfigurationError(
|
|
660
|
-
`Transport '${this.name}' already subscribed; list every topic in one subscription or use one KafkaTransport per consumer group`
|
|
661
|
-
);
|
|
662
|
-
}
|
|
663
|
-
if (this.#groupId === void 0 || this.#groupId.length === 0) {
|
|
664
|
-
throw new core.ConfigurationError(`Transport '${this.name}' needs a groupId to subscribe`);
|
|
665
|
-
}
|
|
666
|
-
if (destinations.length === 0) {
|
|
667
|
-
throw new core.ConfigurationError(`Transport '${this.name}' subscribed to no topics`);
|
|
668
|
-
}
|
|
669
|
-
for (const topic of destinations) {
|
|
670
|
-
const deadLetterTopic = this.#deadLetterTopicFor(topic);
|
|
671
|
-
if (deadLetterTopic !== void 0 && destinations.includes(deadLetterTopic)) {
|
|
672
|
-
throw new core.ConfigurationError(
|
|
673
|
-
`Dead letter topic '${deadLetterTopic}' is also consumed, which would loop`
|
|
674
|
-
);
|
|
675
|
-
}
|
|
676
|
-
}
|
|
677
|
-
}
|
|
678
|
-
/** The dead-letter topic for records of `sourceTopic`, or undefined when the policy does not use one. */
|
|
679
|
-
#deadLetterTopicFor(sourceTopic) {
|
|
680
|
-
const policy = this.#invalidPolicy;
|
|
681
|
-
if (policy === "dead-letter") {
|
|
682
|
-
return `${sourceTopic}.dead`;
|
|
683
|
-
}
|
|
684
|
-
if (typeof policy === "object") {
|
|
685
|
-
const { deadLetterTopic } = policy;
|
|
686
|
-
return typeof deadLetterTopic === "function" ? deadLetterTopic(sourceTopic) : deadLetterTopic;
|
|
687
|
-
}
|
|
688
|
-
return void 0;
|
|
689
|
-
}
|
|
690
|
-
/** Sends with the publish timeout; the late outcome of a timed-out send is ignored. */
|
|
691
|
-
async #send(producer, record) {
|
|
692
|
-
let cancel;
|
|
693
|
-
const timeout = new Promise((_resolve, reject) => {
|
|
694
|
-
cancel = internal.schedule(core.systemClock, {
|
|
695
|
-
run: () => {
|
|
696
|
-
reject(
|
|
697
|
-
new PublishTimeoutError(
|
|
698
|
-
`Transport '${this.name}' got no broker ack for topic '${record.topic}' within ${this.#settings.publishTimeoutMs}ms`
|
|
699
|
-
)
|
|
700
|
-
);
|
|
701
|
-
},
|
|
702
|
-
delayMs: this.#settings.publishTimeoutMs
|
|
703
|
-
});
|
|
704
|
-
});
|
|
705
|
-
const sending = producer.send(record);
|
|
706
|
-
sending.catch(() => void 0);
|
|
707
|
-
try {
|
|
708
|
-
await Promise.race([sending, timeout]);
|
|
709
|
-
} finally {
|
|
710
|
-
cancel?.();
|
|
711
|
-
}
|
|
712
|
-
}
|
|
713
|
-
#connectedProducer() {
|
|
714
|
-
if (this.#state === "closed") {
|
|
715
|
-
return Promise.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
|
|
716
|
-
}
|
|
717
|
-
this.#producer ??= this.#openProducer();
|
|
718
|
-
return this.#producer;
|
|
719
|
-
}
|
|
720
|
-
async #openProducer() {
|
|
721
|
-
const producer = this.#kafka.producer({
|
|
722
|
-
...this.#producerConfig,
|
|
723
|
-
idempotent: true,
|
|
724
|
-
maxInFlightRequests: this.#maxInFlightRequests
|
|
725
|
-
});
|
|
726
|
-
try {
|
|
727
|
-
await producer.connect();
|
|
728
|
-
} catch (error) {
|
|
729
|
-
this.#producer = void 0;
|
|
730
|
-
throw error;
|
|
731
|
-
}
|
|
732
|
-
this.#producerConnected = this.#state !== "closed";
|
|
733
|
-
return producer;
|
|
734
|
-
}
|
|
735
|
-
async #handleMessage(payload, {
|
|
736
|
-
onMessage,
|
|
737
|
-
hooks
|
|
738
|
-
}) {
|
|
944
|
+
async #handleMessage(payload, route) {
|
|
739
945
|
let envelope;
|
|
740
946
|
try {
|
|
741
|
-
envelope = this.#
|
|
947
|
+
envelope = decodeRecord(this.#settings.codec, payload);
|
|
742
948
|
} catch (error) {
|
|
743
|
-
if (this.#codec === cloudEventsBinary && !(error instanceof InvalidKafkaMessageError)) {
|
|
949
|
+
if (this.#settings.codec === cloudEventsBinary && !(error instanceof InvalidKafkaMessageError)) {
|
|
744
950
|
throw toHandlerFailure(error);
|
|
745
951
|
}
|
|
746
|
-
hooks?.onInvalidMessage({ destination: payload.topic, error });
|
|
952
|
+
route.hooks?.onInvalidMessage({ destination: payload.topic, error });
|
|
747
953
|
await this.#withRetries(payload, {
|
|
748
954
|
method: "handleInvalid",
|
|
749
955
|
operation: () => this.#handleInvalid(payload, error)
|
|
@@ -754,7 +960,7 @@ var KafkaTransport = class {
|
|
|
754
960
|
method: "deliver",
|
|
755
961
|
operation: async () => {
|
|
756
962
|
try {
|
|
757
|
-
await onMessage(envelope, { destination: payload.topic });
|
|
963
|
+
await route.onMessage(envelope, { destination: payload.topic });
|
|
758
964
|
} catch (error) {
|
|
759
965
|
if (!(error instanceof core.NonRetryableError)) {
|
|
760
966
|
throw error;
|
|
@@ -764,30 +970,15 @@ var KafkaTransport = class {
|
|
|
764
970
|
}
|
|
765
971
|
});
|
|
766
972
|
}
|
|
767
|
-
#decodeRecord({ topic, partition, message }) {
|
|
768
|
-
if (this.#codec === cloudEventsBinary) {
|
|
769
|
-
return decodeEnvelope(message);
|
|
770
|
-
}
|
|
771
|
-
const record = {
|
|
772
|
-
topic,
|
|
773
|
-
partition,
|
|
774
|
-
offset: message.offset,
|
|
775
|
-
key: message.key,
|
|
776
|
-
value: message.value,
|
|
777
|
-
headers: message.headers,
|
|
778
|
-
timestamp: message.timestamp
|
|
779
|
-
};
|
|
780
|
-
return completeDecodedEvent(this.#codec.decode(record), record);
|
|
781
|
-
}
|
|
782
973
|
async #handleInvalid(payload, error) {
|
|
783
974
|
const { topic, partition, message } = payload;
|
|
784
975
|
const { message: reason, name, code } = core.toErrorInfo(error);
|
|
785
976
|
const location = `topic=${topic} partition=${partition} offset=${message.offset}`;
|
|
786
977
|
const errorFields = `errorName=${name}${code === void 0 ? "" : ` code=${code}`}`;
|
|
787
|
-
const policy = this.#invalidPolicy;
|
|
788
|
-
const deadLetterTopic =
|
|
978
|
+
const policy = this.#settings.invalidPolicy;
|
|
979
|
+
const deadLetterTopic = deadLetterTopicFor(policy, topic);
|
|
789
980
|
if (policy === "skip") {
|
|
790
|
-
this.#logger.error(
|
|
981
|
+
this.#settings.logger.error(
|
|
791
982
|
`[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason} ${errorFields}`
|
|
792
983
|
);
|
|
793
984
|
} else if (typeof policy === "function") {
|
|
@@ -800,32 +991,15 @@ var KafkaTransport = class {
|
|
|
800
991
|
value: message.value
|
|
801
992
|
});
|
|
802
993
|
} else {
|
|
803
|
-
const
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
"relaybus-source-offset": message.offset,
|
|
809
|
-
"relaybus-error-name": name
|
|
810
|
-
};
|
|
811
|
-
if (code === void 0) {
|
|
812
|
-
delete headers["relaybus-error-code"];
|
|
813
|
-
} else {
|
|
814
|
-
headers["relaybus-error-code"] = code;
|
|
815
|
-
}
|
|
816
|
-
const producer = await this.#connectedProducer();
|
|
817
|
-
await this.#send(producer, {
|
|
818
|
-
topic: deadLetterTopic,
|
|
819
|
-
acks: -1,
|
|
820
|
-
messages: [
|
|
821
|
-
{
|
|
822
|
-
key: message.key === null ? null : Buffer.from(message.key),
|
|
823
|
-
value: message.value === null ? null : Buffer.from(message.value),
|
|
824
|
-
headers
|
|
825
|
-
}
|
|
826
|
-
]
|
|
994
|
+
const record = deadLetterRecord(payload, {
|
|
995
|
+
reason,
|
|
996
|
+
name,
|
|
997
|
+
code,
|
|
998
|
+
deadLetterTopic
|
|
827
999
|
});
|
|
828
|
-
this.#
|
|
1000
|
+
const producer = await this.#producer.connected();
|
|
1001
|
+
await this.#producer.send(producer, record);
|
|
1002
|
+
this.#settings.logger.error(
|
|
829
1003
|
`[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason} ${errorFields}`
|
|
830
1004
|
);
|
|
831
1005
|
}
|
|
@@ -836,7 +1010,8 @@ var KafkaTransport = class {
|
|
|
836
1010
|
* redelivers the record; heartbeats between attempts keep the session alive.
|
|
837
1011
|
*/
|
|
838
1012
|
async #withRetries(payload, { method, operation }) {
|
|
839
|
-
const { maxAttempts,
|
|
1013
|
+
const { maxAttempts, processingBackoff } = this.#settings.timings;
|
|
1014
|
+
const { logger } = this.#settings;
|
|
840
1015
|
const location = `topic=${payload.topic} partition=${payload.partition} offset=${payload.message.offset}`;
|
|
841
1016
|
const partitionKey = `${payload.topic}:${payload.partition}`;
|
|
842
1017
|
for (let attempt = 1; ; attempt += 1) {
|
|
@@ -848,51 +1023,28 @@ var KafkaTransport = class {
|
|
|
848
1023
|
const reason = core.toErrorInfo(error).message;
|
|
849
1024
|
if (attempt >= maxAttempts || this.#state === "closed") {
|
|
850
1025
|
this.#stalledPartitions.add(partitionKey);
|
|
851
|
-
|
|
1026
|
+
logger.error(
|
|
852
1027
|
`[KafkaTransport](withRetries) Giving up for now, the record stays uncommitted and will be redelivered | operation=${method} ${location} attempts=${attempt} error=${reason}`
|
|
853
1028
|
);
|
|
854
1029
|
throw toHandlerFailure(error);
|
|
855
1030
|
}
|
|
856
|
-
|
|
1031
|
+
logger.warn(
|
|
857
1032
|
`[KafkaTransport](withRetries) Attempt failed, retrying | operation=${method} ${location} attempt=${attempt} error=${reason}`
|
|
858
1033
|
);
|
|
859
|
-
await this.#
|
|
860
|
-
|
|
861
|
-
payload
|
|
862
|
-
|
|
1034
|
+
await this.#sleeps.sleepInSlices(processingBackoff.delayFor(attempt), {
|
|
1035
|
+
sliceMs: this.#settings.timings.heartbeatIntervalMs,
|
|
1036
|
+
between: () => payload.heartbeat(),
|
|
1037
|
+
stopped: () => this.#state === "closed"
|
|
1038
|
+
});
|
|
863
1039
|
}
|
|
864
1040
|
}
|
|
865
1041
|
}
|
|
866
|
-
/** Sleeps in slices and heartbeats after each, so a long backoff never outlives the session timeout. */
|
|
867
|
-
async #sleepWithHeartbeats(milliseconds, payload) {
|
|
868
|
-
let remaining = milliseconds;
|
|
869
|
-
do {
|
|
870
|
-
const slice = Math.min(remaining, this.#settings.heartbeatIntervalMs);
|
|
871
|
-
await this.#sleep(slice);
|
|
872
|
-
await payload.heartbeat();
|
|
873
|
-
remaining -= slice;
|
|
874
|
-
} while (remaining > 0 && this.#state !== "closed");
|
|
875
|
-
}
|
|
876
|
-
#sleep(milliseconds) {
|
|
877
|
-
return new Promise((resolve) => {
|
|
878
|
-
const wake = () => {
|
|
879
|
-
cancel();
|
|
880
|
-
this.#sleepers.delete(wake);
|
|
881
|
-
resolve();
|
|
882
|
-
};
|
|
883
|
-
const cancel = internal.schedule(core.systemClock, { run: wake, delayMs: milliseconds });
|
|
884
|
-
this.#sleepers.add(wake);
|
|
885
|
-
});
|
|
886
|
-
}
|
|
887
|
-
/** Whether the client exposes the instrumentation needed to follow the group membership. */
|
|
888
|
-
#tracksMembership(consumer) {
|
|
889
|
-
return consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0;
|
|
890
|
-
}
|
|
891
1042
|
#observeConsumer(consumer) {
|
|
892
|
-
if (!
|
|
1043
|
+
if (!tracksMembership(consumer)) {
|
|
893
1044
|
this.#groupJoined = true;
|
|
894
1045
|
return;
|
|
895
1046
|
}
|
|
1047
|
+
const { logger, groupId } = this.#settings;
|
|
896
1048
|
const names = consumer.events;
|
|
897
1049
|
const listen = (name, listener) => {
|
|
898
1050
|
if (name !== void 0) {
|
|
@@ -901,29 +1053,28 @@ var KafkaTransport = class {
|
|
|
901
1053
|
};
|
|
902
1054
|
listen(names.GROUP_JOIN, () => {
|
|
903
1055
|
this.#groupJoined = true;
|
|
904
|
-
this.#
|
|
905
|
-
|
|
1056
|
+
this.#groupJoin.joined();
|
|
1057
|
+
logger.log(`[KafkaTransport](observeConsumer) Joined group | group=${groupId}`);
|
|
906
1058
|
});
|
|
907
1059
|
listen(names.REBALANCING, () => {
|
|
908
1060
|
this.#groupJoined = false;
|
|
909
1061
|
this.#stalledPartitions.clear();
|
|
910
|
-
|
|
1062
|
+
logger.warn(`[KafkaTransport](observeConsumer) Rebalancing | group=${groupId}`);
|
|
911
1063
|
});
|
|
912
1064
|
listen(names.CRASH, (event) => this.#handleCrash(event));
|
|
913
|
-
|
|
914
|
-
this.#groupJoined = false;
|
|
915
|
-
});
|
|
916
|
-
listen(names.DISCONNECT, () => {
|
|
1065
|
+
const leftGroup = () => {
|
|
917
1066
|
this.#groupJoined = false;
|
|
918
|
-
}
|
|
1067
|
+
};
|
|
1068
|
+
listen(names.STOP, leftGroup);
|
|
1069
|
+
listen(names.DISCONNECT, leftGroup);
|
|
919
1070
|
}
|
|
920
1071
|
#handleCrash(event) {
|
|
921
1072
|
const { error, restart } = event.payload ?? {};
|
|
922
1073
|
this.#groupJoined = false;
|
|
923
|
-
this.#logger.error(
|
|
924
|
-
`[KafkaTransport](handleCrash) Consumer crashed | group=${this.#groupId} restart=${String(restart)} error=${core.toErrorInfo(error).message}`
|
|
1074
|
+
this.#settings.logger.error(
|
|
1075
|
+
`[KafkaTransport](handleCrash) Consumer crashed | group=${this.#settings.groupId} restart=${String(restart)} error=${core.toErrorInfo(error).message}`
|
|
925
1076
|
);
|
|
926
|
-
this.#
|
|
1077
|
+
this.#groupJoin.failed(error);
|
|
927
1078
|
if (restart === false && this.#consuming && this.#restartLoop === void 0) {
|
|
928
1079
|
this.#restartLoop = this.#restartConsumer().finally(() => {
|
|
929
1080
|
this.#restartLoop = void 0;
|
|
@@ -935,125 +1086,28 @@ var KafkaTransport = class {
|
|
|
935
1086
|
* again, with bounded backoff, until it joined or the transport closed.
|
|
936
1087
|
*/
|
|
937
1088
|
async #restartConsumer() {
|
|
938
|
-
const {
|
|
1089
|
+
const { logger, groupId, timings } = this.#settings;
|
|
939
1090
|
const consumer = this.#consumer;
|
|
940
1091
|
for (let attempt = 1; this.#state === "subscribed"; attempt += 1) {
|
|
941
|
-
await this.#sleep(
|
|
942
|
-
Math.min(restartInitialBackoffMs * 2 ** (attempt - 1), restartMaxBackoffMs)
|
|
943
|
-
);
|
|
1092
|
+
await this.#sleeps.sleep(timings.restartBackoff.delayFor(attempt));
|
|
944
1093
|
if (this.#state !== "subscribed") {
|
|
945
1094
|
return;
|
|
946
1095
|
}
|
|
947
1096
|
await consumer.disconnect().catch(() => void 0);
|
|
948
1097
|
try {
|
|
949
1098
|
await this.#startConsuming();
|
|
950
|
-
|
|
951
|
-
`[KafkaTransport](restartConsumer) Consumer restarted | group=${
|
|
1099
|
+
logger.log(
|
|
1100
|
+
`[KafkaTransport](restartConsumer) Consumer restarted | group=${groupId} attempt=${attempt}`
|
|
952
1101
|
);
|
|
953
1102
|
return;
|
|
954
1103
|
} catch (error) {
|
|
955
|
-
|
|
956
|
-
`[KafkaTransport](restartConsumer) Restart failed, will retry | group=${
|
|
1104
|
+
logger.error(
|
|
1105
|
+
`[KafkaTransport](restartConsumer) Restart failed, will retry | group=${groupId} attempt=${attempt} error=${core.toErrorInfo(error).message}`
|
|
957
1106
|
);
|
|
958
1107
|
}
|
|
959
1108
|
}
|
|
960
1109
|
}
|
|
961
|
-
#waitForGroupJoin(consumer) {
|
|
962
|
-
if (!this.#tracksMembership(consumer)) {
|
|
963
|
-
return { promise: Promise.resolve(), dispose: () => void 0 };
|
|
964
|
-
}
|
|
965
|
-
let cancel;
|
|
966
|
-
const promise = new Promise((resolve, reject) => {
|
|
967
|
-
this.#joinWaiter = { resolve, reject };
|
|
968
|
-
cancel = internal.schedule(core.systemClock, {
|
|
969
|
-
run: () => {
|
|
970
|
-
reject(
|
|
971
|
-
new GroupJoinTimeoutError(
|
|
972
|
-
`Consumer of transport '${this.name}' did not join group '${this.#groupId}' within ${this.#settings.startupTimeoutMs}ms`
|
|
973
|
-
)
|
|
974
|
-
);
|
|
975
|
-
},
|
|
976
|
-
delayMs: this.#settings.startupTimeoutMs
|
|
977
|
-
});
|
|
978
|
-
});
|
|
979
|
-
promise.catch(() => void 0);
|
|
980
|
-
return {
|
|
981
|
-
promise,
|
|
982
|
-
dispose: () => {
|
|
983
|
-
cancel?.();
|
|
984
|
-
this.#joinWaiter = void 0;
|
|
985
|
-
}
|
|
986
|
-
};
|
|
987
|
-
}
|
|
988
1110
|
};
|
|
989
|
-
function resolveCodec({ codec, decode }) {
|
|
990
|
-
if (decode === void 0) {
|
|
991
|
-
return codec ?? cloudEventsBinary;
|
|
992
|
-
}
|
|
993
|
-
if (codec !== void 0) {
|
|
994
|
-
throw new core.ConfigurationError("Give 'codec' or the deprecated 'decode', not both");
|
|
995
|
-
}
|
|
996
|
-
return { encode: encodeEnvelope, decode };
|
|
997
|
-
}
|
|
998
|
-
function resolveSettings(options) {
|
|
999
|
-
const processing = options.processing ?? {};
|
|
1000
|
-
const maxAttempts = internal.assertPositiveInteger(
|
|
1001
|
-
processing.maxAttempts ?? DEFAULTS.maxAttempts,
|
|
1002
|
-
"processing.maxAttempts"
|
|
1003
|
-
);
|
|
1004
|
-
return {
|
|
1005
|
-
maxAttempts,
|
|
1006
|
-
initialBackoffMs: core.toMilliseconds(
|
|
1007
|
-
processing.initialBackoff ?? DEFAULTS.initialBackoff,
|
|
1008
|
-
"processing.initialBackoff"
|
|
1009
|
-
),
|
|
1010
|
-
maxBackoffMs: core.toMilliseconds(
|
|
1011
|
-
processing.maxBackoff ?? DEFAULTS.maxBackoff,
|
|
1012
|
-
"processing.maxBackoff"
|
|
1013
|
-
),
|
|
1014
|
-
startupTimeoutMs: core.toMilliseconds(
|
|
1015
|
-
options.startupTimeout ?? DEFAULTS.startupTimeout,
|
|
1016
|
-
"startupTimeout"
|
|
1017
|
-
),
|
|
1018
|
-
publishTimeoutMs: core.toMilliseconds(
|
|
1019
|
-
options.publishTimeout ?? DEFAULTS.publishTimeout,
|
|
1020
|
-
"publishTimeout"
|
|
1021
|
-
),
|
|
1022
|
-
heartbeatIntervalMs: Math.max(
|
|
1023
|
-
1,
|
|
1024
|
-
core.toMilliseconds(
|
|
1025
|
-
processing.heartbeatInterval ?? DEFAULTS.heartbeatInterval,
|
|
1026
|
-
"processing.heartbeatInterval"
|
|
1027
|
-
)
|
|
1028
|
-
),
|
|
1029
|
-
...resolveRestartSettings(options.restart ?? {})
|
|
1030
|
-
};
|
|
1031
|
-
}
|
|
1032
|
-
function resolveRestartSettings(restart) {
|
|
1033
|
-
return {
|
|
1034
|
-
restartInitialBackoffMs: core.toMilliseconds(
|
|
1035
|
-
restart.initialBackoff ?? DEFAULTS.restartInitialBackoff,
|
|
1036
|
-
"restart.initialBackoff"
|
|
1037
|
-
),
|
|
1038
|
-
restartMaxBackoffMs: core.toMilliseconds(
|
|
1039
|
-
restart.maxBackoff ?? DEFAULTS.restartMaxBackoff,
|
|
1040
|
-
"restart.maxBackoff"
|
|
1041
|
-
)
|
|
1042
|
-
};
|
|
1043
|
-
}
|
|
1044
|
-
var MAXIMUM_IN_FLIGHT_REQUESTS = 5;
|
|
1045
|
-
function assertMaxInFlightRequests(value) {
|
|
1046
|
-
if (!Number.isInteger(value) || value < 1 || value > MAXIMUM_IN_FLIGHT_REQUESTS) {
|
|
1047
|
-
throw new core.ConfigurationError(
|
|
1048
|
-
`maxInFlightRequests must be an integer from 1 to 5, received ${value}`
|
|
1049
|
-
);
|
|
1050
|
-
}
|
|
1051
|
-
return value;
|
|
1052
|
-
}
|
|
1053
|
-
function toHandlerFailure(error) {
|
|
1054
|
-
const { name, message } = core.toErrorInfo(error);
|
|
1055
|
-
return new HandlerFailedError(`${name}: ${message}`);
|
|
1056
|
-
}
|
|
1057
1111
|
|
|
1058
1112
|
Object.defineProperty(exports, "fromHeader", {
|
|
1059
1113
|
enumerable: true,
|