@relaybus/kafka 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -312,25 +312,17 @@ function plainJsonDecoder(options) {
312
312
  if (record.headers?.[SPEC_VERSION_HEADER] !== void 0 || internal.mediaTypeOf(headers[CONTENT_TYPE]) === STRUCTURED_MEDIA_TYPE2) {
313
313
  return decodeCloudEvent(record);
314
314
  }
315
- const plain = {
316
- topic: record.topic,
317
- partition: record.partition,
318
- offset: record.offset,
319
- key: readKeyLeniently(record.key),
320
- headers,
321
- payload: parsePayload(record.value),
322
- timestamp: record.timestamp
323
- };
324
- const type = resolveText(options.type, { record: plain, name: "type" });
315
+ const plain = plainRecordOf(record, headers);
316
+ const type = KAFKA_EVENTS.text(options.type, { plain, name: "type" });
325
317
  if (type === void 0) {
326
318
  throw invalid("Record has no event type: the type option found none");
327
319
  }
328
320
  assertValidType(type, "The event type");
329
321
  return {
330
- id: resolveText(options.id, { record: plain, name: "id" }) ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
322
+ id: KAFKA_EVENTS.text(options.id, { plain, name: "id" }) ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
331
323
  type,
332
- source: resolveText(options.source, { record: plain, name: "source" }) ?? record.topic,
333
- key: options.key === void 0 ? plain.key : resolveText(options.key, { record: plain, name: "key" }) ?? null,
324
+ source: KAFKA_EVENTS.text(options.source, { plain, name: "source" }) ?? record.topic,
325
+ key: options.key === void 0 ? plain.key : KAFKA_EVENTS.text(options.key, { plain, name: "key" }) ?? null,
334
326
  payload: plain.payload,
335
327
  headers,
336
328
  correlationId: null,
@@ -339,27 +331,38 @@ function plainJsonDecoder(options) {
339
331
  };
340
332
  };
341
333
  }
342
- function completeDecodedEvent(decoded, record) {
343
- if (typeof decoded !== "object" || decoded === null) {
344
- throw invalid("The decoder must return an event object synchronously");
345
- }
346
- if ("then" in decoded) {
347
- observeRejection(decoded);
348
- throw invalid("The decoder must return an event object synchronously");
349
- }
350
- const event = decoded;
334
+ function plainRecordOf(record, headers) {
351
335
  return {
352
- id: event.id,
353
- type: event.type,
354
- source: event.source ?? record.topic,
355
- key: event.key === void 0 ? readRecordKey(record.key) : event.key,
356
- payload: event.payload,
357
- headers: event.headers ?? {},
358
- correlationId: event.correlationId ?? null,
359
- causationId: event.causationId ?? null,
360
- occurredAt: event.occurredAt ?? requireRecordTimestamp(record)
336
+ topic: record.topic,
337
+ partition: record.partition,
338
+ offset: record.offset,
339
+ key: readKeyLeniently(record.key),
340
+ headers,
341
+ payload: parsePayload(record.value),
342
+ timestamp: record.timestamp
361
343
  };
362
344
  }
345
+ var KafkaDecodedEvents = class extends internal.DecodedEventReader {
346
+ invalid(reason) {
347
+ return invalid(reason);
348
+ }
349
+ /** The topic. */
350
+ defaultSource(record) {
351
+ return record.topic;
352
+ }
353
+ /** The record key as UTF-8 text. */
354
+ defaultKey(record) {
355
+ return readRecordKey(record.key);
356
+ }
357
+ /** The record timestamp; a record without a usable one is malformed. */
358
+ defaultTime(record) {
359
+ return requireRecordTimestamp(record);
360
+ }
361
+ };
362
+ var KAFKA_EVENTS = new KafkaDecodedEvents();
363
+ function completeDecodedEvent(decoded, record) {
364
+ return KAFKA_EVENTS.complete(decoded, record);
365
+ }
363
366
  function extensionAttributes(body) {
364
367
  return Object.fromEntries(
365
368
  Object.entries(body).filter(
@@ -405,33 +408,17 @@ function readKeyLeniently(key) {
405
408
  return Buffer.from(key).toString("base64");
406
409
  }
407
410
  }
408
- function resolveText(option, { record, name }) {
409
- const value = typeof option === "function" ? option(record) : option;
410
- if (value === void 0 || value === null) {
411
- return void 0;
412
- }
413
- if (typeof value !== "string") {
414
- observeRejection(value);
415
- throw invalid(`The ${name} option returned ${typeof value}, expected a string`);
416
- }
417
- return value.length === 0 ? void 0 : value;
418
- }
419
411
  function resolveOccurredAt(option, { plain, record }) {
420
412
  const value = option?.(plain);
421
413
  if (value === void 0) {
422
414
  return requireRecordTimestamp(record);
423
415
  }
424
416
  if (!(value instanceof Date) || Number.isNaN(value.getTime())) {
425
- observeRejection(value);
417
+ internal.observeRejection(value);
426
418
  throw invalid("The occurredAt option returned no valid Date");
427
419
  }
428
420
  return value;
429
421
  }
430
- function observeRejection(value) {
431
- if (typeof value === "object" && value !== null && "then" in value) {
432
- Promise.resolve(value).catch(() => void 0);
433
- }
434
- }
435
422
 
436
423
  // src/codecs.ts
437
424
  var cloudEventsBinary = Object.freeze({
@@ -454,32 +441,34 @@ function jsonEnvelope(mapping) {
454
441
  const stampsTime = mapping.occurredAt === void 0;
455
442
  const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { [CONTENT_TYPE]: contentType, [EVENT_ID_HEADER]: envelope.id } : { [CONTENT_TYPE]: contentType };
456
443
  return {
457
- encode: (envelope) => {
458
- assertRepresentable(envelope);
459
- const { body, headers } = format.write(envelope);
460
- const taken = [CONTENT_TYPE, EVENT_ID_HEADER].find((name) => Object.hasOwn(headers, name));
461
- if (taken !== void 0) {
462
- throw new UnencodableEventError(
463
- `Event ${envelope.id} has a header '${taken}', which the codec writes itself`
464
- );
465
- }
466
- return {
467
- key: envelope.key,
468
- value: serializeJson(envelope, body),
469
- headers: { ...headers, ...ownHeaders(envelope) },
470
- ...stampsTime ? { timestamp: recordTimestamp(envelope) } : {}
471
- };
472
- },
473
- decode: (record) => {
474
- const { [EVENT_ID_HEADER]: eventId, ...headers } = withoutContentType(
475
- readHeadersLeniently(record.headers)
476
- );
477
- const fields = format.read({ body: parsePayload(record.value), headers });
478
- return {
479
- ...fields,
480
- id: fields.id ?? (eventId === void 0 || eventId === "" ? `kafka:${record.topic}:${record.partition}:${record.offset}` : eventId)
481
- };
482
- }
444
+ encode: (envelope) => encodeJson(envelope, { format, stampsTime, ownHeaders }),
445
+ decode: (record) => decodeJson(record, format)
446
+ };
447
+ }
448
+ function encodeJson(envelope, { format, stampsTime, ownHeaders }) {
449
+ assertRepresentable(envelope);
450
+ const { body, headers } = format.write(envelope);
451
+ const taken = [CONTENT_TYPE, EVENT_ID_HEADER].find((name) => Object.hasOwn(headers, name));
452
+ if (taken !== void 0) {
453
+ throw new UnencodableEventError(
454
+ `Event ${envelope.id} has a header '${taken}', which the codec writes itself`
455
+ );
456
+ }
457
+ return {
458
+ key: envelope.key,
459
+ value: serializeJson(envelope, body),
460
+ headers: { ...headers, ...ownHeaders(envelope) },
461
+ ...stampsTime ? { timestamp: recordTimestamp(envelope) } : {}
462
+ };
463
+ }
464
+ function decodeJson(record, format) {
465
+ const { [EVENT_ID_HEADER]: eventId, ...headers } = withoutContentType(
466
+ readHeadersLeniently(record.headers)
467
+ );
468
+ const fields = format.read({ body: parsePayload(record.value), headers });
469
+ return {
470
+ ...fields,
471
+ id: fields.id ?? (eventId === void 0 || eventId === "" ? `kafka:${record.topic}:${record.partition}:${record.offset}` : eventId)
483
472
  };
484
473
  }
485
474
  function withoutContentType(headers) {
@@ -497,6 +486,139 @@ function recordTimestamp(envelope) {
497
486
  formatTime(envelope);
498
487
  return String(envelope.occurredAt.getTime());
499
488
  }
489
+ function tracksMembership(consumer) {
490
+ return consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0;
491
+ }
492
+ var GroupJoinWatch = class {
493
+ #settings;
494
+ #waiter;
495
+ constructor(settings) {
496
+ this.#settings = settings;
497
+ }
498
+ /** The consumer joined its group. */
499
+ joined() {
500
+ this.#waiter?.resolve();
501
+ }
502
+ /** The consumer crashed while it was joining. */
503
+ failed(error) {
504
+ this.#waiter?.reject(error);
505
+ }
506
+ /** The transport closed while the consumer was joining. */
507
+ closed() {
508
+ this.#waiter?.reject(new TransportClosedError(`Transport '${this.#settings.name}' is closed`));
509
+ }
510
+ /** Connects and subscribes the consumer, runs it and resolves once it joined its group. */
511
+ async start(consumer, {
512
+ topics,
513
+ fromBeginning,
514
+ eachMessage
515
+ }) {
516
+ await consumer.connect();
517
+ await consumer.subscribe({ topics, fromBeginning });
518
+ const joined = this.#wait(consumer);
519
+ try {
520
+ await consumer.run({ eachMessage });
521
+ await joined.promise;
522
+ } finally {
523
+ joined.dispose();
524
+ }
525
+ }
526
+ #wait(consumer) {
527
+ if (!tracksMembership(consumer)) {
528
+ return { promise: Promise.resolve(), dispose: () => void 0 };
529
+ }
530
+ const { name, groupId, timings } = this.#settings;
531
+ let cancel;
532
+ const promise = new Promise((resolve, reject) => {
533
+ this.#waiter = { resolve, reject };
534
+ cancel = internal.schedule(core.systemClock, {
535
+ run: () => {
536
+ reject(
537
+ new GroupJoinTimeoutError(
538
+ `Consumer of transport '${name}' did not join group '${groupId}' within ${timings.startupTimeoutMs}ms`
539
+ )
540
+ );
541
+ },
542
+ delayMs: timings.startupTimeoutMs
543
+ });
544
+ });
545
+ promise.catch(() => void 0);
546
+ return {
547
+ promise,
548
+ dispose: () => {
549
+ cancel?.();
550
+ this.#waiter = void 0;
551
+ }
552
+ };
553
+ }
554
+ };
555
+ var KafkaProducerSession = class {
556
+ #settings;
557
+ /** Whether the transport is closed: then nothing connects any more. */
558
+ #isClosed;
559
+ #producer;
560
+ #connected = false;
561
+ constructor(settings, isClosed) {
562
+ this.#settings = settings;
563
+ this.#isClosed = isClosed;
564
+ }
565
+ /** True once the producer connected, until the transport closes. */
566
+ get isConnected() {
567
+ return this.#connected;
568
+ }
569
+ /** The producer opened so far, if any: what closing must disconnect. */
570
+ get pending() {
571
+ return this.#producer;
572
+ }
573
+ markClosed() {
574
+ this.#connected = false;
575
+ }
576
+ connected() {
577
+ if (this.#isClosed()) {
578
+ return Promise.reject(
579
+ new TransportClosedError(`Transport '${this.#settings.name}' is closed`)
580
+ );
581
+ }
582
+ this.#producer ??= this.#open();
583
+ return this.#producer;
584
+ }
585
+ /** Sends with the publish timeout; the late outcome of a timed-out send is ignored. */
586
+ async send(producer, record) {
587
+ const { name, timings } = this.#settings;
588
+ let cancel;
589
+ const timeout = new Promise((_resolve, reject) => {
590
+ cancel = internal.schedule(core.systemClock, {
591
+ run: () => {
592
+ reject(
593
+ new PublishTimeoutError(
594
+ `Transport '${name}' got no broker ack for topic '${record.topic}' within ${timings.publishTimeoutMs}ms`
595
+ )
596
+ );
597
+ },
598
+ delayMs: timings.publishTimeoutMs
599
+ });
600
+ });
601
+ const sending = producer.send(record);
602
+ sending.catch(() => void 0);
603
+ try {
604
+ await Promise.race([sending, timeout]);
605
+ } finally {
606
+ cancel?.();
607
+ }
608
+ }
609
+ async #open() {
610
+ const { kafka, producerConfig, maxInFlightRequests } = this.#settings;
611
+ const producer = kafka.producer({ ...producerConfig, idempotent: true, maxInFlightRequests });
612
+ try {
613
+ await producer.connect();
614
+ } catch (error) {
615
+ this.#producer = void 0;
616
+ throw error;
617
+ }
618
+ this.#connected = !this.#isClosed();
619
+ return producer;
620
+ }
621
+ };
500
622
  var DEFAULTS = {
501
623
  name: "kafka",
502
624
  maxInFlightRequests: 1,
@@ -509,44 +631,224 @@ var DEFAULTS = {
509
631
  restartInitialBackoff: "1s",
510
632
  restartMaxBackoff: "30s"
511
633
  };
512
- var KafkaTransport = class {
634
+ var MAXIMUM_IN_FLIGHT_REQUESTS = 5;
635
+ function resolveKafkaSettings(options) {
636
+ return {
637
+ name: options.name ?? DEFAULTS.name,
638
+ kafka: options.kafka,
639
+ groupId: options.groupId,
640
+ fromBeginning: options.fromBeginning ?? true,
641
+ maxInFlightRequests: internal.BoundedInteger.of(
642
+ options.maxInFlightRequests ?? DEFAULTS.maxInFlightRequests,
643
+ { name: "maxInFlightRequests", minimum: 1, maximum: MAXIMUM_IN_FLIGHT_REQUESTS }
644
+ ),
645
+ producerConfig: options.producerConfig ?? {},
646
+ consumerConfig: options.consumerConfig ?? {},
647
+ invalidPolicy: options.onInvalidMessage ?? "dead-letter",
648
+ codec: resolveCodec(options),
649
+ logger: options.logger ?? console,
650
+ timings: resolveTimings(options)
651
+ };
652
+ }
653
+ function resolveCodec({ codec, decode }) {
654
+ if (decode === void 0) {
655
+ return codec ?? cloudEventsBinary;
656
+ }
657
+ if (codec !== void 0) {
658
+ throw new core.ConfigurationError("Give 'codec' or the deprecated 'decode', not both");
659
+ }
660
+ return { encode: encodeEnvelope, decode };
661
+ }
662
+ function resolveTimings(options) {
663
+ const processing = options.processing ?? {};
664
+ const maxAttempts = internal.PositiveInteger.of(
665
+ processing.maxAttempts ?? DEFAULTS.maxAttempts,
666
+ "processing.maxAttempts"
667
+ );
668
+ return {
669
+ maxAttempts,
670
+ processingBackoff: new internal.ExponentialBackoff({
671
+ initialMs: internal.Milliseconds.of(
672
+ processing.initialBackoff ?? DEFAULTS.initialBackoff,
673
+ "processing.initialBackoff"
674
+ ),
675
+ maxMs: internal.Milliseconds.of(processing.maxBackoff ?? DEFAULTS.maxBackoff, "processing.maxBackoff")
676
+ }),
677
+ startupTimeoutMs: internal.Milliseconds.of(
678
+ options.startupTimeout ?? DEFAULTS.startupTimeout,
679
+ "startupTimeout"
680
+ ),
681
+ publishTimeoutMs: internal.Milliseconds.of(
682
+ options.publishTimeout ?? DEFAULTS.publishTimeout,
683
+ "publishTimeout"
684
+ ),
685
+ heartbeatIntervalMs: Math.max(
686
+ 1,
687
+ internal.Milliseconds.of(
688
+ processing.heartbeatInterval ?? DEFAULTS.heartbeatInterval,
689
+ "processing.heartbeatInterval"
690
+ )
691
+ ),
692
+ restartBackoff: resolveRestartBackoff(options.restart ?? {})
693
+ };
694
+ }
695
+ function resolveRestartBackoff(restart) {
696
+ return new internal.ExponentialBackoff({
697
+ initialMs: internal.Milliseconds.of(
698
+ restart.initialBackoff ?? DEFAULTS.restartInitialBackoff,
699
+ "restart.initialBackoff"
700
+ ),
701
+ maxMs: internal.Milliseconds.of(restart.maxBackoff ?? DEFAULTS.restartMaxBackoff, "restart.maxBackoff")
702
+ });
703
+ }
704
+ function deadLetterTopicFor(policy, sourceTopic) {
705
+ if (policy === "dead-letter") {
706
+ return `${sourceTopic}.dead`;
707
+ }
708
+ if (typeof policy === "object") {
709
+ const { deadLetterTopic } = policy;
710
+ return typeof deadLetterTopic === "function" ? deadLetterTopic(sourceTopic) : deadLetterTopic;
711
+ }
712
+ return void 0;
713
+ }
714
+ function toHandlerFailure(error) {
715
+ const { name, message } = core.toErrorInfo(error);
716
+ return new HandlerFailedError(`${name}: ${message}`);
717
+ }
718
+ function assertCanSubscribe(state, {
719
+ settings,
720
+ destinations
721
+ }) {
722
+ const { name, groupId, invalidPolicy } = settings;
723
+ if (state === "closed") {
724
+ throw new TransportClosedError(`Transport '${name}' is closed`);
725
+ }
726
+ if (state === "subscribed") {
727
+ throw new core.ConfigurationError(
728
+ `Transport '${name}' already subscribed; list every topic in one subscription or use one KafkaTransport per consumer group`
729
+ );
730
+ }
731
+ if (groupId === void 0 || groupId.length === 0) {
732
+ throw new core.ConfigurationError(`Transport '${name}' needs a groupId to subscribe`);
733
+ }
734
+ if (destinations.length === 0) {
735
+ throw new core.ConfigurationError(`Transport '${name}' subscribed to no topics`);
736
+ }
737
+ for (const topic of destinations) {
738
+ const deadLetterTopic = deadLetterTopicFor(invalidPolicy, topic);
739
+ if (deadLetterTopic !== void 0 && destinations.includes(deadLetterTopic)) {
740
+ throw new core.ConfigurationError(
741
+ `Dead letter topic '${deadLetterTopic}' is also consumed, which would loop`
742
+ );
743
+ }
744
+ }
745
+ }
746
+
747
+ // src/kafka-records.ts
748
+ function decodeRecord(codec, { topic, partition, message }) {
749
+ if (codec === cloudEventsBinary) {
750
+ return decodeEnvelope(message);
751
+ }
752
+ const record = {
753
+ topic,
754
+ partition,
755
+ offset: message.offset,
756
+ key: message.key,
757
+ value: message.value,
758
+ headers: message.headers,
759
+ timestamp: message.timestamp
760
+ };
761
+ return completeDecodedEvent(codec.decode(record), record);
762
+ }
763
+ function deadLetterRecord({ topic, partition, message }, {
764
+ reason,
765
+ name,
766
+ code,
767
+ deadLetterTopic
768
+ }) {
769
+ const headers = {
770
+ ...toOutgoingHeaders(message.headers),
771
+ "relaybus-invalid-reason": reason,
772
+ "relaybus-source-topic": topic,
773
+ "relaybus-source-partition": String(partition),
774
+ "relaybus-source-offset": message.offset,
775
+ "relaybus-error-name": name
776
+ };
777
+ if (code === void 0) {
778
+ delete headers["relaybus-error-code"];
779
+ } else {
780
+ headers["relaybus-error-code"] = code;
781
+ }
782
+ return {
783
+ topic: deadLetterTopic,
784
+ acks: -1,
785
+ messages: [
786
+ {
787
+ key: message.key === null ? null : Buffer.from(message.key),
788
+ value: message.value === null ? null : Buffer.from(message.value),
789
+ headers
790
+ }
791
+ ]
792
+ };
793
+ }
794
+ var WakeableSleeps = class {
795
+ #sleepers = /* @__PURE__ */ new Set();
796
+ sleep(milliseconds) {
797
+ return new Promise((resolve) => {
798
+ const wake = () => {
799
+ cancel();
800
+ this.#sleepers.delete(wake);
801
+ resolve();
802
+ };
803
+ const cancel = internal.schedule(core.systemClock, { run: wake, delayMs: milliseconds });
804
+ this.#sleepers.add(wake);
805
+ });
806
+ }
807
+ wakeAll() {
808
+ for (const wake of this.#sleepers) {
809
+ wake();
810
+ }
811
+ }
812
+ /**
813
+ * Sleeps `milliseconds` in slices of at most `sliceMs`, running `between` after each, until done
814
+ * or `stopped`: a long backoff then never outlives the session timeout.
815
+ */
816
+ async sleepInSlices(milliseconds, {
817
+ sliceMs,
818
+ between,
819
+ stopped
820
+ }) {
821
+ let remaining = milliseconds;
822
+ do {
823
+ const slice = Math.min(remaining, sliceMs);
824
+ await this.sleep(slice);
825
+ await between();
826
+ remaining -= slice;
827
+ } while (remaining > 0 && !stopped());
828
+ }
829
+ };
830
+
831
+ // src/kafka-transport.ts
832
+ var KafkaTransport = class extends internal.SubscribingTransport {
513
833
  name;
514
834
  system = "kafka";
515
- #kafka;
516
- #groupId;
517
- #fromBeginning;
518
- #maxInFlightRequests;
519
- #producerConfig;
520
- #consumerConfig;
521
- #invalidPolicy;
522
- #codec;
523
- #logger;
524
835
  #settings;
525
- #sleepers = /* @__PURE__ */ new Set();
526
- #state = "idle";
527
836
  #producer;
837
+ #groupJoin;
838
+ #sleeps = new WakeableSleeps();
839
+ #state = "idle";
528
840
  #consumer;
529
841
  #groupJoined = false;
530
842
  #consuming = false;
531
- #producerConnected = false;
532
843
  #restartLoop;
533
844
  #stalledPartitions = /* @__PURE__ */ new Set();
534
845
  #startConsuming;
535
- #joinWaiter;
536
846
  constructor(options) {
537
- this.name = options.name ?? DEFAULTS.name;
538
- this.#kafka = options.kafka;
539
- this.#groupId = options.groupId;
540
- this.#fromBeginning = options.fromBeginning ?? true;
541
- this.#maxInFlightRequests = assertMaxInFlightRequests(
542
- options.maxInFlightRequests ?? DEFAULTS.maxInFlightRequests
543
- );
544
- this.#producerConfig = options.producerConfig ?? {};
545
- this.#consumerConfig = options.consumerConfig ?? {};
546
- this.#invalidPolicy = options.onInvalidMessage ?? "dead-letter";
547
- this.#codec = resolveCodec(options);
548
- this.#logger = options.logger ?? console;
549
- this.#settings = resolveSettings(options);
847
+ super();
848
+ this.#settings = resolveKafkaSettings(options);
849
+ this.name = this.#settings.name;
850
+ this.#producer = new KafkaProducerSession(this.#settings, () => this.#state === "closed");
851
+ this.#groupJoin = new GroupJoinWatch(this.#settings);
550
852
  }
551
853
  /**
552
854
  * True while the consumer is joined to its group, not between a crash or
@@ -563,21 +865,21 @@ var KafkaTransport = class {
563
865
  * dead consumer is being restarted.
564
866
  */
565
867
  isReady() {
566
- if (this.#state === "closed" || !this.#producerConnected) {
868
+ if (this.#state === "closed" || !this.#producer.isConnected) {
567
869
  return false;
568
870
  }
569
871
  return this.#state === "idle" || this.isConsuming();
570
872
  }
571
873
  /** Connects the producer. Called by the bus on start; `publish` also connects on demand. */
572
874
  async connect() {
573
- await this.#connectedProducer();
875
+ await this.#producer.connected();
574
876
  }
575
877
  async publish(envelope, destination) {
576
- const producer = await this.#connectedProducer();
577
- await this.#send(producer, {
878
+ const producer = await this.#producer.connected();
879
+ await this.#producer.send(producer, {
578
880
  topic: destination,
579
881
  acks: -1,
580
- messages: [this.#codec.encode(envelope)]
882
+ messages: [this.#settings.codec.encode(envelope)]
581
883
  });
582
884
  }
583
885
  /**
@@ -587,14 +889,21 @@ var KafkaTransport = class {
587
889
  * @throws ConfigurationError when `groupId` is missing, `destinations` is empty
588
890
  * or the transport already subscribed (create one transport per subscription).
589
891
  */
590
- // eslint-disable-next-line relaybus/max-two-params -- the Transport.subscribe signature
591
- async subscribe(destinations, onMessage, hooks) {
592
- this.#assertCanSubscribe(destinations);
892
+ openSubscription(subscription) {
893
+ return this.#subscribe(subscription);
894
+ }
895
+ async #subscribe({ destinations, onMessage, hooks }) {
896
+ assertCanSubscribe(this.#state, { settings: this.#settings, destinations });
593
897
  this.#state = "subscribed";
594
- const consumer = this.#kafka.consumer({ ...this.#consumerConfig, groupId: this.#groupId });
898
+ const { kafka, consumerConfig, groupId } = this.#settings;
899
+ const consumer = kafka.consumer({ ...consumerConfig, groupId });
595
900
  this.#consumer = consumer;
596
901
  this.#observeConsumer(consumer);
597
- const start = () => this.#startConsumer(consumer, { destinations: [...destinations], onMessage, hooks });
902
+ const start = () => this.#groupJoin.start(consumer, {
903
+ topics: [...destinations],
904
+ fromBeginning: this.#settings.fromBeginning,
905
+ eachMessage: (payload) => this.#handleMessage(payload, { onMessage, hooks })
906
+ });
598
907
  this.#startConsuming = start;
599
908
  try {
600
909
  await start();
@@ -606,40 +915,21 @@ var KafkaTransport = class {
606
915
  throw error;
607
916
  }
608
917
  this.#consuming = true;
609
- this.#logger.log(
610
- `[KafkaTransport](subscribe) Consuming | transport=${this.name} group=${this.#groupId} topics=${destinations.join(",")}`
918
+ this.#settings.logger.log(
919
+ `[KafkaTransport](subscribe) Consuming | transport=${this.name} group=${groupId} topics=${destinations.join(",")}`
611
920
  );
612
921
  }
613
- async #startConsumer(consumer, {
614
- destinations,
615
- onMessage,
616
- hooks
617
- }) {
618
- await consumer.connect();
619
- await consumer.subscribe({ topics: destinations, fromBeginning: this.#fromBeginning });
620
- const joined = this.#waitForGroupJoin(consumer);
621
- try {
622
- await consumer.run({
623
- eachMessage: (payload) => this.#handleMessage(payload, { onMessage, hooks })
624
- });
625
- await joined.promise;
626
- } finally {
627
- joined.dispose();
628
- }
629
- }
630
922
  /** Stops consuming and disconnects the consumer and the producer. Safe to call twice. */
631
923
  async close() {
632
924
  if (this.#state === "closed") {
633
925
  return;
634
926
  }
635
927
  this.#state = "closed";
636
- const pendingProducer = this.#producer;
928
+ const pendingProducer = this.#producer.pending;
637
929
  this.#groupJoined = false;
638
- this.#producerConnected = false;
639
- for (const wake of this.#sleepers) {
640
- wake();
641
- }
642
- this.#joinWaiter?.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
930
+ this.#producer.markClosed();
931
+ this.#sleeps.wakeAll();
932
+ this.#groupJoin.closed();
643
933
  await this.#restartLoop;
644
934
  const producer = await pendingProducer?.catch(() => void 0);
645
935
  const results = await Promise.allSettled([
@@ -651,99 +941,15 @@ var KafkaTransport = class {
651
941
  throw failure.reason;
652
942
  }
653
943
  }
654
- #assertCanSubscribe(destinations) {
655
- if (this.#state === "closed") {
656
- throw new TransportClosedError(`Transport '${this.name}' is closed`);
657
- }
658
- if (this.#state === "subscribed") {
659
- throw new core.ConfigurationError(
660
- `Transport '${this.name}' already subscribed; list every topic in one subscription or use one KafkaTransport per consumer group`
661
- );
662
- }
663
- if (this.#groupId === void 0 || this.#groupId.length === 0) {
664
- throw new core.ConfigurationError(`Transport '${this.name}' needs a groupId to subscribe`);
665
- }
666
- if (destinations.length === 0) {
667
- throw new core.ConfigurationError(`Transport '${this.name}' subscribed to no topics`);
668
- }
669
- for (const topic of destinations) {
670
- const deadLetterTopic = this.#deadLetterTopicFor(topic);
671
- if (deadLetterTopic !== void 0 && destinations.includes(deadLetterTopic)) {
672
- throw new core.ConfigurationError(
673
- `Dead letter topic '${deadLetterTopic}' is also consumed, which would loop`
674
- );
675
- }
676
- }
677
- }
678
- /** The dead-letter topic for records of `sourceTopic`, or undefined when the policy does not use one. */
679
- #deadLetterTopicFor(sourceTopic) {
680
- const policy = this.#invalidPolicy;
681
- if (policy === "dead-letter") {
682
- return `${sourceTopic}.dead`;
683
- }
684
- if (typeof policy === "object") {
685
- const { deadLetterTopic } = policy;
686
- return typeof deadLetterTopic === "function" ? deadLetterTopic(sourceTopic) : deadLetterTopic;
687
- }
688
- return void 0;
689
- }
690
- /** Sends with the publish timeout; the late outcome of a timed-out send is ignored. */
691
- async #send(producer, record) {
692
- let cancel;
693
- const timeout = new Promise((_resolve, reject) => {
694
- cancel = internal.schedule(core.systemClock, {
695
- run: () => {
696
- reject(
697
- new PublishTimeoutError(
698
- `Transport '${this.name}' got no broker ack for topic '${record.topic}' within ${this.#settings.publishTimeoutMs}ms`
699
- )
700
- );
701
- },
702
- delayMs: this.#settings.publishTimeoutMs
703
- });
704
- });
705
- const sending = producer.send(record);
706
- sending.catch(() => void 0);
707
- try {
708
- await Promise.race([sending, timeout]);
709
- } finally {
710
- cancel?.();
711
- }
712
- }
713
- #connectedProducer() {
714
- if (this.#state === "closed") {
715
- return Promise.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
716
- }
717
- this.#producer ??= this.#openProducer();
718
- return this.#producer;
719
- }
720
- async #openProducer() {
721
- const producer = this.#kafka.producer({
722
- ...this.#producerConfig,
723
- idempotent: true,
724
- maxInFlightRequests: this.#maxInFlightRequests
725
- });
726
- try {
727
- await producer.connect();
728
- } catch (error) {
729
- this.#producer = void 0;
730
- throw error;
731
- }
732
- this.#producerConnected = this.#state !== "closed";
733
- return producer;
734
- }
735
- async #handleMessage(payload, {
736
- onMessage,
737
- hooks
738
- }) {
944
+ async #handleMessage(payload, route) {
739
945
  let envelope;
740
946
  try {
741
- envelope = this.#decodeRecord(payload);
947
+ envelope = decodeRecord(this.#settings.codec, payload);
742
948
  } catch (error) {
743
- if (this.#codec === cloudEventsBinary && !(error instanceof InvalidKafkaMessageError)) {
949
+ if (this.#settings.codec === cloudEventsBinary && !(error instanceof InvalidKafkaMessageError)) {
744
950
  throw toHandlerFailure(error);
745
951
  }
746
- hooks?.onInvalidMessage({ destination: payload.topic, error });
952
+ route.hooks?.onInvalidMessage({ destination: payload.topic, error });
747
953
  await this.#withRetries(payload, {
748
954
  method: "handleInvalid",
749
955
  operation: () => this.#handleInvalid(payload, error)
@@ -754,7 +960,7 @@ var KafkaTransport = class {
754
960
  method: "deliver",
755
961
  operation: async () => {
756
962
  try {
757
- await onMessage(envelope, { destination: payload.topic });
963
+ await route.onMessage(envelope, { destination: payload.topic });
758
964
  } catch (error) {
759
965
  if (!(error instanceof core.NonRetryableError)) {
760
966
  throw error;
@@ -764,30 +970,15 @@ var KafkaTransport = class {
764
970
  }
765
971
  });
766
972
  }
767
- #decodeRecord({ topic, partition, message }) {
768
- if (this.#codec === cloudEventsBinary) {
769
- return decodeEnvelope(message);
770
- }
771
- const record = {
772
- topic,
773
- partition,
774
- offset: message.offset,
775
- key: message.key,
776
- value: message.value,
777
- headers: message.headers,
778
- timestamp: message.timestamp
779
- };
780
- return completeDecodedEvent(this.#codec.decode(record), record);
781
- }
782
973
  async #handleInvalid(payload, error) {
783
974
  const { topic, partition, message } = payload;
784
975
  const { message: reason, name, code } = core.toErrorInfo(error);
785
976
  const location = `topic=${topic} partition=${partition} offset=${message.offset}`;
786
977
  const errorFields = `errorName=${name}${code === void 0 ? "" : ` code=${code}`}`;
787
- const policy = this.#invalidPolicy;
788
- const deadLetterTopic = this.#deadLetterTopicFor(topic);
978
+ const policy = this.#settings.invalidPolicy;
979
+ const deadLetterTopic = deadLetterTopicFor(policy, topic);
789
980
  if (policy === "skip") {
790
- this.#logger.error(
981
+ this.#settings.logger.error(
791
982
  `[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason} ${errorFields}`
792
983
  );
793
984
  } else if (typeof policy === "function") {
@@ -800,32 +991,15 @@ var KafkaTransport = class {
800
991
  value: message.value
801
992
  });
802
993
  } else {
803
- const headers = {
804
- ...toOutgoingHeaders(message.headers),
805
- "relaybus-invalid-reason": reason,
806
- "relaybus-source-topic": topic,
807
- "relaybus-source-partition": String(partition),
808
- "relaybus-source-offset": message.offset,
809
- "relaybus-error-name": name
810
- };
811
- if (code === void 0) {
812
- delete headers["relaybus-error-code"];
813
- } else {
814
- headers["relaybus-error-code"] = code;
815
- }
816
- const producer = await this.#connectedProducer();
817
- await this.#send(producer, {
818
- topic: deadLetterTopic,
819
- acks: -1,
820
- messages: [
821
- {
822
- key: message.key === null ? null : Buffer.from(message.key),
823
- value: message.value === null ? null : Buffer.from(message.value),
824
- headers
825
- }
826
- ]
994
+ const record = deadLetterRecord(payload, {
995
+ reason,
996
+ name,
997
+ code,
998
+ deadLetterTopic
827
999
  });
828
- this.#logger.error(
1000
+ const producer = await this.#producer.connected();
1001
+ await this.#producer.send(producer, record);
1002
+ this.#settings.logger.error(
829
1003
  `[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason} ${errorFields}`
830
1004
  );
831
1005
  }
@@ -836,7 +1010,8 @@ var KafkaTransport = class {
836
1010
  * redelivers the record; heartbeats between attempts keep the session alive.
837
1011
  */
838
1012
  async #withRetries(payload, { method, operation }) {
839
- const { maxAttempts, initialBackoffMs, maxBackoffMs } = this.#settings;
1013
+ const { maxAttempts, processingBackoff } = this.#settings.timings;
1014
+ const { logger } = this.#settings;
840
1015
  const location = `topic=${payload.topic} partition=${payload.partition} offset=${payload.message.offset}`;
841
1016
  const partitionKey = `${payload.topic}:${payload.partition}`;
842
1017
  for (let attempt = 1; ; attempt += 1) {
@@ -848,51 +1023,28 @@ var KafkaTransport = class {
848
1023
  const reason = core.toErrorInfo(error).message;
849
1024
  if (attempt >= maxAttempts || this.#state === "closed") {
850
1025
  this.#stalledPartitions.add(partitionKey);
851
- this.#logger.error(
1026
+ logger.error(
852
1027
  `[KafkaTransport](withRetries) Giving up for now, the record stays uncommitted and will be redelivered | operation=${method} ${location} attempts=${attempt} error=${reason}`
853
1028
  );
854
1029
  throw toHandlerFailure(error);
855
1030
  }
856
- this.#logger.warn(
1031
+ logger.warn(
857
1032
  `[KafkaTransport](withRetries) Attempt failed, retrying | operation=${method} ${location} attempt=${attempt} error=${reason}`
858
1033
  );
859
- await this.#sleepWithHeartbeats(
860
- Math.min(initialBackoffMs * 2 ** (attempt - 1), maxBackoffMs),
861
- payload
862
- );
1034
+ await this.#sleeps.sleepInSlices(processingBackoff.delayFor(attempt), {
1035
+ sliceMs: this.#settings.timings.heartbeatIntervalMs,
1036
+ between: () => payload.heartbeat(),
1037
+ stopped: () => this.#state === "closed"
1038
+ });
863
1039
  }
864
1040
  }
865
1041
  }
866
- /** Sleeps in slices and heartbeats after each, so a long backoff never outlives the session timeout. */
867
- async #sleepWithHeartbeats(milliseconds, payload) {
868
- let remaining = milliseconds;
869
- do {
870
- const slice = Math.min(remaining, this.#settings.heartbeatIntervalMs);
871
- await this.#sleep(slice);
872
- await payload.heartbeat();
873
- remaining -= slice;
874
- } while (remaining > 0 && this.#state !== "closed");
875
- }
876
- #sleep(milliseconds) {
877
- return new Promise((resolve) => {
878
- const wake = () => {
879
- cancel();
880
- this.#sleepers.delete(wake);
881
- resolve();
882
- };
883
- const cancel = internal.schedule(core.systemClock, { run: wake, delayMs: milliseconds });
884
- this.#sleepers.add(wake);
885
- });
886
- }
887
- /** Whether the client exposes the instrumentation needed to follow the group membership. */
888
- #tracksMembership(consumer) {
889
- return consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0;
890
- }
891
1042
  #observeConsumer(consumer) {
892
- if (!this.#tracksMembership(consumer)) {
1043
+ if (!tracksMembership(consumer)) {
893
1044
  this.#groupJoined = true;
894
1045
  return;
895
1046
  }
1047
+ const { logger, groupId } = this.#settings;
896
1048
  const names = consumer.events;
897
1049
  const listen = (name, listener) => {
898
1050
  if (name !== void 0) {
@@ -901,29 +1053,28 @@ var KafkaTransport = class {
901
1053
  };
902
1054
  listen(names.GROUP_JOIN, () => {
903
1055
  this.#groupJoined = true;
904
- this.#joinWaiter?.resolve();
905
- this.#logger.log(`[KafkaTransport](observeConsumer) Joined group | group=${this.#groupId}`);
1056
+ this.#groupJoin.joined();
1057
+ logger.log(`[KafkaTransport](observeConsumer) Joined group | group=${groupId}`);
906
1058
  });
907
1059
  listen(names.REBALANCING, () => {
908
1060
  this.#groupJoined = false;
909
1061
  this.#stalledPartitions.clear();
910
- this.#logger.warn(`[KafkaTransport](observeConsumer) Rebalancing | group=${this.#groupId}`);
1062
+ logger.warn(`[KafkaTransport](observeConsumer) Rebalancing | group=${groupId}`);
911
1063
  });
912
1064
  listen(names.CRASH, (event) => this.#handleCrash(event));
913
- listen(names.STOP, () => {
914
- this.#groupJoined = false;
915
- });
916
- listen(names.DISCONNECT, () => {
1065
+ const leftGroup = () => {
917
1066
  this.#groupJoined = false;
918
- });
1067
+ };
1068
+ listen(names.STOP, leftGroup);
1069
+ listen(names.DISCONNECT, leftGroup);
919
1070
  }
920
1071
  #handleCrash(event) {
921
1072
  const { error, restart } = event.payload ?? {};
922
1073
  this.#groupJoined = false;
923
- this.#logger.error(
924
- `[KafkaTransport](handleCrash) Consumer crashed | group=${this.#groupId} restart=${String(restart)} error=${core.toErrorInfo(error).message}`
1074
+ this.#settings.logger.error(
1075
+ `[KafkaTransport](handleCrash) Consumer crashed | group=${this.#settings.groupId} restart=${String(restart)} error=${core.toErrorInfo(error).message}`
925
1076
  );
926
- this.#joinWaiter?.reject(error);
1077
+ this.#groupJoin.failed(error);
927
1078
  if (restart === false && this.#consuming && this.#restartLoop === void 0) {
928
1079
  this.#restartLoop = this.#restartConsumer().finally(() => {
929
1080
  this.#restartLoop = void 0;
@@ -935,125 +1086,28 @@ var KafkaTransport = class {
935
1086
  * again, with bounded backoff, until it joined or the transport closed.
936
1087
  */
937
1088
  async #restartConsumer() {
938
- const { restartInitialBackoffMs, restartMaxBackoffMs } = this.#settings;
1089
+ const { logger, groupId, timings } = this.#settings;
939
1090
  const consumer = this.#consumer;
940
1091
  for (let attempt = 1; this.#state === "subscribed"; attempt += 1) {
941
- await this.#sleep(
942
- Math.min(restartInitialBackoffMs * 2 ** (attempt - 1), restartMaxBackoffMs)
943
- );
1092
+ await this.#sleeps.sleep(timings.restartBackoff.delayFor(attempt));
944
1093
  if (this.#state !== "subscribed") {
945
1094
  return;
946
1095
  }
947
1096
  await consumer.disconnect().catch(() => void 0);
948
1097
  try {
949
1098
  await this.#startConsuming();
950
- this.#logger.log(
951
- `[KafkaTransport](restartConsumer) Consumer restarted | group=${this.#groupId} attempt=${attempt}`
1099
+ logger.log(
1100
+ `[KafkaTransport](restartConsumer) Consumer restarted | group=${groupId} attempt=${attempt}`
952
1101
  );
953
1102
  return;
954
1103
  } catch (error) {
955
- this.#logger.error(
956
- `[KafkaTransport](restartConsumer) Restart failed, will retry | group=${this.#groupId} attempt=${attempt} error=${core.toErrorInfo(error).message}`
1104
+ logger.error(
1105
+ `[KafkaTransport](restartConsumer) Restart failed, will retry | group=${groupId} attempt=${attempt} error=${core.toErrorInfo(error).message}`
957
1106
  );
958
1107
  }
959
1108
  }
960
1109
  }
961
- #waitForGroupJoin(consumer) {
962
- if (!this.#tracksMembership(consumer)) {
963
- return { promise: Promise.resolve(), dispose: () => void 0 };
964
- }
965
- let cancel;
966
- const promise = new Promise((resolve, reject) => {
967
- this.#joinWaiter = { resolve, reject };
968
- cancel = internal.schedule(core.systemClock, {
969
- run: () => {
970
- reject(
971
- new GroupJoinTimeoutError(
972
- `Consumer of transport '${this.name}' did not join group '${this.#groupId}' within ${this.#settings.startupTimeoutMs}ms`
973
- )
974
- );
975
- },
976
- delayMs: this.#settings.startupTimeoutMs
977
- });
978
- });
979
- promise.catch(() => void 0);
980
- return {
981
- promise,
982
- dispose: () => {
983
- cancel?.();
984
- this.#joinWaiter = void 0;
985
- }
986
- };
987
- }
988
1110
  };
989
- function resolveCodec({ codec, decode }) {
990
- if (decode === void 0) {
991
- return codec ?? cloudEventsBinary;
992
- }
993
- if (codec !== void 0) {
994
- throw new core.ConfigurationError("Give 'codec' or the deprecated 'decode', not both");
995
- }
996
- return { encode: encodeEnvelope, decode };
997
- }
998
- function resolveSettings(options) {
999
- const processing = options.processing ?? {};
1000
- const maxAttempts = internal.assertPositiveInteger(
1001
- processing.maxAttempts ?? DEFAULTS.maxAttempts,
1002
- "processing.maxAttempts"
1003
- );
1004
- return {
1005
- maxAttempts,
1006
- initialBackoffMs: core.toMilliseconds(
1007
- processing.initialBackoff ?? DEFAULTS.initialBackoff,
1008
- "processing.initialBackoff"
1009
- ),
1010
- maxBackoffMs: core.toMilliseconds(
1011
- processing.maxBackoff ?? DEFAULTS.maxBackoff,
1012
- "processing.maxBackoff"
1013
- ),
1014
- startupTimeoutMs: core.toMilliseconds(
1015
- options.startupTimeout ?? DEFAULTS.startupTimeout,
1016
- "startupTimeout"
1017
- ),
1018
- publishTimeoutMs: core.toMilliseconds(
1019
- options.publishTimeout ?? DEFAULTS.publishTimeout,
1020
- "publishTimeout"
1021
- ),
1022
- heartbeatIntervalMs: Math.max(
1023
- 1,
1024
- core.toMilliseconds(
1025
- processing.heartbeatInterval ?? DEFAULTS.heartbeatInterval,
1026
- "processing.heartbeatInterval"
1027
- )
1028
- ),
1029
- ...resolveRestartSettings(options.restart ?? {})
1030
- };
1031
- }
1032
- function resolveRestartSettings(restart) {
1033
- return {
1034
- restartInitialBackoffMs: core.toMilliseconds(
1035
- restart.initialBackoff ?? DEFAULTS.restartInitialBackoff,
1036
- "restart.initialBackoff"
1037
- ),
1038
- restartMaxBackoffMs: core.toMilliseconds(
1039
- restart.maxBackoff ?? DEFAULTS.restartMaxBackoff,
1040
- "restart.maxBackoff"
1041
- )
1042
- };
1043
- }
1044
- var MAXIMUM_IN_FLIGHT_REQUESTS = 5;
1045
- function assertMaxInFlightRequests(value) {
1046
- if (!Number.isInteger(value) || value < 1 || value > MAXIMUM_IN_FLIGHT_REQUESTS) {
1047
- throw new core.ConfigurationError(
1048
- `maxInFlightRequests must be an integer from 1 to 5, received ${value}`
1049
- );
1050
- }
1051
- return value;
1052
- }
1053
- function toHandlerFailure(error) {
1054
- const { name, message } = core.toErrorInfo(error);
1055
- return new HandlerFailedError(`${name}: ${message}`);
1056
- }
1057
1111
 
1058
1112
  Object.defineProperty(exports, "fromHeader", {
1059
1113
  enumerable: true,