@relaybus/kafka 0.3.1 → 0.4.1

package/dist/index.js

@@ -1,4 +1,4 @@
-import { RelayBusError, NonRetryableError, assertEventType, toErrorInfo, systemClock, ConfigurationError } from '@relaybus/core';
+import { RelayBusError, NonRetryableError, assertEventType, ConfigurationError, toErrorInfo, systemClock } from '@relaybus/core';
 import { createCloudEventReaders, DecodedEventReader, mediaTypeOf, isJsonObject, observeRejection, createJsonEnvelopeFormat, SubscribingTransport, writeStructuredCloudEvent, schedule, BoundedInteger, PositiveInteger, ExponentialBackoff, Milliseconds } from '@relaybus/core/internal';
 export { fromHeader, fromPayloadField } from '@relaybus/core/internal';
 
@@ -436,15 +436,30 @@ function jsonEnvelope(mapping) {
     transportIds: [RECORD_POSITION],
     failures: { invalid, unencodable: (reason) => new UnencodableEventError(reason) }
   });
-  const contentType = mapping.contentType ?? DEFAULT_CONTENT_TYPE;
+  const contentType = mapping.contentTypeHeader === false ? {} : { [CONTENT_TYPE]: mapping.contentType ?? DEFAULT_CONTENT_TYPE };
   const stampsTime = mapping.occurredAt === void 0;
-  const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { [CONTENT_TYPE]: contentType, [EVENT_ID_HEADER]: envelope.id } : { [CONTENT_TYPE]: contentType };
+  const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { ...contentType, [EVENT_ID_HEADER]: envelope.id } : contentType;
+  const recordKey = resolveRecordKey(mapping.recordKey);
   return {
-    encode: (envelope) => encodeJson(envelope, { format, stampsTime, ownHeaders }),
+    encode: (envelope) => encodeJson(envelope, { format, stampsTime, ownHeaders, recordKey }),
     decode: (record) => decodeJson(record, format)
   };
 }
-function encodeJson(envelope, { format, stampsTime, ownHeaders }) {
+function resolveRecordKey(recordKey) {
+  if (recordKey === void 0 || recordKey === "event-key") {
+    return (envelope) => envelope.key;
+  }
+  if (recordKey === "none") {
+    return () => null;
+  }
+  if (typeof recordKey === "function") {
+    return recordKey;
+  }
+  throw new ConfigurationError(
+    `jsonEnvelope recordKey must be 'event-key', 'none' or a function, received '${String(recordKey)}'`
+  );
+}
+function encodeJson(envelope, { format, stampsTime, ownHeaders, recordKey }) {
   assertRepresentable(envelope);
   const { body, headers } = format.write(envelope);
   const taken = [CONTENT_TYPE, EVENT_ID_HEADER].find((name) => Object.hasOwn(headers, name));
@@ -454,7 +469,7 @@ function encodeJson(envelope, { format, stampsTime, ownHeaders }) {
     );
   }
   return {
-    key: envelope.key,
+    key: recordKey(envelope),
     value: serializeJson(envelope, body),
     headers: { ...headers, ...ownHeaders(envelope) },
     ...stampsTime ? { timestamp: recordTimestamp(envelope) } : {}
@@ -485,9 +500,6 @@ function recordTimestamp(envelope) {
   formatTime(envelope);
   return String(envelope.occurredAt.getTime());
 }
-function tracksMembership(consumer) {
-  return consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0;
-}
 var GroupJoinWatch = class {
   #settings;
   #waiter;
@@ -513,7 +525,7 @@ var GroupJoinWatch = class {
     eachMessage
   }) {
     await consumer.connect();
-    await consumer.subscribe({ topics, fromBeginning });
+    await consumer.subscribe(this.#settings.dialect.subscription(topics, fromBeginning));
     const joined = this.#wait(consumer);
     try {
       await consumer.run({ eachMessage });
@@ -523,7 +535,7 @@ var GroupJoinWatch = class {
     }
   }
   #wait(consumer) {
-    if (!tracksMembership(consumer)) {
+    if (!this.#settings.dialect.tracksMembership(consumer)) {
       return { promise: Promise.resolve(), dispose: () => void 0 };
     }
     const { name, groupId, timings } = this.#settings;
@@ -597,7 +609,7 @@ var KafkaProducerSession = class {
         delayMs: timings.publishTimeoutMs
       });
     });
-    const sending = producer.send(record);
+    const sending = producer.send(this.#settings.dialect.record(record));
     sending.catch(() => void 0);
     try {
       await Promise.race([sending, timeout]);
@@ -606,8 +618,12 @@ var KafkaProducerSession = class {
     }
   }
   async #open() {
-    const { kafka, producerConfig, maxInFlightRequests } = this.#settings;
-    const producer = kafka.producer({ ...producerConfig, idempotent: true, maxInFlightRequests });
+    const { kafka, dialect, injectedProducer } = this.#settings;
+    if (injectedProducer !== void 0) {
+      this.#connected = true;
+      return withoutLifecycle(injectedProducer);
+    }
+    const producer = kafka.producer(dialect.producerConfig(this.#settings));
     try {
       await producer.connect();
     } catch (error) {
@@ -618,6 +634,88 @@ var KafkaProducerSession = class {
     return producer;
   }
 };
+function withoutLifecycle(producer) {
+  return {
+    send: (record) => producer.send(record),
+    disconnect: () => Promise.resolve()
+  };
+}
+var KAFKAJS_DIALECT = {
+  producerConfig: ({ producerConfig, maxInFlightRequests }) => ({
+    ...producerConfig,
+    idempotent: true,
+    maxInFlightRequests
+  }),
+  consumerConfig: ({ consumerConfig, groupId }) => ({ ...consumerConfig, groupId }),
+  subscription: (topics, fromBeginning) => ({ topics, fromBeginning }),
+  record: (record) => record,
+  tracksMembership: (consumer) => consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0,
+  observe: (consumer, membership) => {
+    if (!KAFKAJS_DIALECT.tracksMembership(consumer)) {
+      return;
+    }
+    const names = consumer.events;
+    const listen = (name, listener) => {
+      if (name !== void 0) {
+        consumer.on(name, listener);
+      }
+    };
+    listen(names.GROUP_JOIN, () => membership.joined());
+    listen(names.REBALANCING, () => membership.rebalancing());
+    listen(names.CRASH, (event) => membership.crashed(event));
+    listen(names.STOP, () => membership.left());
+    listen(names.DISCONNECT, () => membership.left());
+  }
+};
+var ASSIGN_PARTITIONS = -175;
+var REVOKE_PARTITIONS = -174;
+var CONFLUENT_DIALECT = {
+  producerConfig: ({ producerConfig, maxInFlightRequests }) => {
+    const user = producerConfig;
+    return {
+      ...user,
+      kafkaJS: { ...user.kafkaJS, idempotent: true, maxInFlightRequests, acks: -1 },
+      "enable.idempotence": true,
+      "max.in.flight": maxInFlightRequests,
+      acks: -1
+    };
+  },
+  consumerConfig: ({ consumerConfig, groupId, fromBeginning }, membership) => {
+    const user = consumerConfig;
+    const userCallback = typeof user.rebalance_cb === "function" ? user.rebalance_cb : void 0;
+    return {
+      ...user,
+      kafkaJS: { ...user.kafkaJS, groupId, fromBeginning },
+      "group.id": groupId,
+      "auto.offset.reset": fromBeginning ? "earliest" : "latest",
+      rebalance_cb: (...args) => {
+        if (args[0].code === ASSIGN_PARTITIONS) {
+          membership.joined();
+        } else if (args[0].code === REVOKE_PARTITIONS) {
+          membership.rebalancing();
+        }
+        return userCallback?.(...args);
+      }
+    };
+  },
+  subscription: (topics) => ({ topics }),
+  record: ({ acks: _acks, ...record }) => record,
+  tracksMembership: () => true,
+  observe: () => void 0
+};
+function clientDialect(client) {
+  if (client === void 0 || client === "kafkajs") {
+    return KAFKAJS_DIALECT;
+  }
+  if (client === "confluent") {
+    return CONFLUENT_DIALECT;
+  }
+  throw new ConfigurationError(
+    `Unknown Kafka client '${String(client)}'; use 'kafkajs' or 'confluent'`
+  );
+}
+
+// src/kafka-settings.ts
 var DEFAULTS = {
   name: "kafka",
   maxInFlightRequests: 1,
@@ -635,6 +733,8 @@ function resolveKafkaSettings(options) {
   return {
     name: options.name ?? DEFAULTS.name,
     kafka: options.kafka,
+    injectedProducer: resolveInjectedProducer(options),
+    dialect: clientDialect(options.client),
     groupId: options.groupId,
     fromBeginning: options.fromBeginning ?? true,
     maxInFlightRequests: BoundedInteger.of(
@@ -649,6 +749,23 @@ function resolveKafkaSettings(options) {
     timings: resolveTimings(options)
   };
 }
+function resolveInjectedProducer(options) {
+  const { producer, kafka, producerConfig, maxInFlightRequests } = options;
+  if (producer === void 0) {
+    if (kafka === void 0) {
+      throw new ConfigurationError(
+        "Give 'kafka', or an already-connected 'producer' to publish with"
+      );
+    }
+    return void 0;
+  }
+  if (producerConfig !== void 0 || maxInFlightRequests !== void 0) {
+    throw new ConfigurationError(
+      "Give 'producerConfig' and 'maxInFlightRequests' to a producer you inject yourself"
+    );
+  }
+  return producer;
+}
 function resolveCodec({ codec, decode }) {
   if (decode === void 0) {
     return codec ?? cloudEventsBinary;
@@ -718,7 +835,7 @@ function assertCanSubscribe(state, {
   settings,
   destinations
 }) {
-  const { name, groupId, invalidPolicy } = settings;
+  const { name, groupId, invalidPolicy, kafka } = settings;
   if (state === "closed") {
     throw new TransportClosedError(`Transport '${name}' is closed`);
   }
@@ -727,6 +844,9 @@ function assertCanSubscribe(state, {
       `Transport '${name}' already subscribed; list every topic in one subscription or use one KafkaTransport per consumer group`
     );
   }
+  if (kafka === void 0) {
+    throw new ConfigurationError(`Transport '${name}' needs 'kafka' to create its consumer`);
+  }
   if (groupId === void 0 || groupId.length === 0) {
     throw new ConfigurationError(`Transport '${name}' needs a groupId to subscribe`);
   }
@@ -745,10 +865,7 @@ function assertCanSubscribe(state, {
 
 // src/kafka-records.ts
 function decodeRecord(codec, { topic, partition, message }) {
-  if (codec === cloudEventsBinary) {
-    return decodeEnvelope(message);
-  }
-  const record = {
+  return decodeKafkaRecord(codec, {
     topic,
     partition,
     offset: message.offset,
@@ -756,7 +873,12 @@ function decodeRecord(codec, { topic, partition, message }) {
     value: message.value,
     headers: message.headers,
     timestamp: message.timestamp
-  };
+  });
+}
+function decodeKafkaRecord(codec, record) {
+  if (codec === cloudEventsBinary) {
+    return decodeEnvelope(record);
+  }
   return completeDecodedEvent(codec.decode(record), record);
 }
 function deadLetterRecord({ topic, partition, message }, {
@@ -894,10 +1016,12 @@ var KafkaTransport = class extends SubscribingTransport {
   async #subscribe({ destinations, onMessage, hooks }) {
     assertCanSubscribe(this.#state, { settings: this.#settings, destinations });
     this.#state = "subscribed";
-    const { kafka, consumerConfig, groupId } = this.#settings;
-    const consumer = kafka.consumer({ ...consumerConfig, groupId });
+    const { kafka, dialect, groupId } = this.#settings;
+    const membership = this.#observeConsumer();
+    const consumer = kafka.consumer(dialect.consumerConfig(this.#settings, membership));
     this.#consumer = consumer;
-    this.#observeConsumer(consumer);
+    this.#groupJoined = !dialect.tracksMembership(consumer);
+    dialect.observe(consumer, membership);
     const start = () => this.#groupJoin.start(consumer, {
       topics: [...destinations],
       fromBeginning: this.#settings.fromBeginning,
@@ -1038,34 +1162,25 @@ var KafkaTransport = class extends SubscribingTransport {
       }
     }
   }
-  #observeConsumer(consumer) {
-    if (!tracksMembership(consumer)) {
-      this.#groupJoined = true;
-      return;
-    }
+  /** What the transport does when the consumer joins, rebalances, crashes or leaves its group. */
+  #observeConsumer() {
     const { logger, groupId } = this.#settings;
-    const names = consumer.events;
-    const listen = (name, listener) => {
-      if (name !== void 0) {
-        consumer.on(name, listener);
+    return {
+      joined: () => {
+        this.#groupJoined = true;
+        this.#groupJoin.joined();
+        logger.log(`[KafkaTransport](observeConsumer) Joined group | group=${groupId}`);
+      },
+      rebalancing: () => {
+        this.#groupJoined = false;
+        this.#stalledPartitions.clear();
+        logger.warn(`[KafkaTransport](observeConsumer) Rebalancing | group=${groupId}`);
+      },
+      crashed: (event) => this.#handleCrash(event),
+      left: () => {
+        this.#groupJoined = false;
       }
     };
-    listen(names.GROUP_JOIN, () => {
-      this.#groupJoined = true;
-      this.#groupJoin.joined();
-      logger.log(`[KafkaTransport](observeConsumer) Joined group | group=${groupId}`);
-    });
-    listen(names.REBALANCING, () => {
-      this.#groupJoined = false;
-      this.#stalledPartitions.clear();
-      logger.warn(`[KafkaTransport](observeConsumer) Rebalancing | group=${groupId}`);
-    });
-    listen(names.CRASH, (event) => this.#handleCrash(event));
-    const leftGroup = () => {
-      this.#groupJoined = false;
-    };
-    listen(names.STOP, leftGroup);
-    listen(names.DISCONNECT, leftGroup);
   }
   #handleCrash(event) {
     const { error, restart } = event.payload ?? {};
@@ -1108,6 +1223,6 @@ var KafkaTransport = class extends SubscribingTransport {
   }
 };
 
-export { GroupJoinTimeoutError, HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, TransportClosedError, UnencodableEventError, cloudEventsBinary, cloudEventsStructured, decodeCloudEvent, decodeEnvelope, encodeEnvelope, jsonEnvelope, plainJsonDecoder };
+export { GroupJoinTimeoutError, HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, TransportClosedError, UnencodableEventError, cloudEventsBinary, cloudEventsStructured, decodeCloudEvent, decodeEnvelope, decodeKafkaRecord, encodeEnvelope, jsonEnvelope, plainJsonDecoder };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map