@relaybus/kafka 0.1.0 → 0.2.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { RelayBusError, NonRetryableError, EventEnvelope, Duration, Transport, RelayBusLogger, TransportMessageHandler } from '@relaybus/core';
+import { RelayBusError, NonRetryableError, EventEnvelope, EventHeaders, Duration, Transport, RelayBusLogger, TransportMessageHandler } from '@relaybus/core';
 
 /**
  * Structural views of the Kafka client API used by this package.
@@ -143,6 +143,125 @@ declare function encodeEnvelope(envelope: EventEnvelope): EncodedMessage;
  */
 declare function decodeEnvelope(message: DecodableMessage): EventEnvelope;
 
+/** A consumed record as handed to a {@link KafkaMessageDecoder}. */
+interface KafkaInboundRecord extends DecodableMessage {
+    readonly topic: string;
+    readonly partition: number;
+    readonly offset: string;
+}
+/**
+ * The event a decoder extracted from a record. `id`, `type` and `payload` are
+ * required; every other field may be left out and is then filled by the
+ * transport: `source` with the topic, `key` with the record key (UTF-8),
+ * `headers` with none, `correlationId` and `causationId` with null, and
+ * `occurredAt` with the record timestamp.
+ */
+interface DecodedEvent {
+    readonly id: string;
+    readonly type: string;
+    readonly payload: unknown;
+    readonly source?: string;
+    readonly key?: string | null;
+    readonly headers?: EventHeaders;
+    readonly correlationId?: string | null;
+    readonly causationId?: string | null;
+    readonly occurredAt?: Date;
+}
+/**
+ * Turns a consumed record into an event, for topics written by producers that
+ * do not use relaybus. Must be synchronous and free of side effects: it runs
+ * again on every redelivery, so the same record must always give the same
+ * event, above all the same `id`, which is what the inbox deduplicates on.
+ *
+ * Anything it throws marks the record as malformed: it goes to the transport's
+ * `onInvalidMessage` policy (dead-letter by default) and the consumer moves on.
+ * An event the bus then rejects (empty id, invalid type or time) is
+ * dead-lettered the same way.
+ */
+type KafkaMessageDecoder = (record: KafkaInboundRecord) => DecodedEvent;
+/** A plain JSON record, decoded for the extractor options of {@link plainJsonDecoder}. */
+interface PlainJsonRecord {
+    readonly topic: string;
+    readonly partition: number;
+    readonly offset: string;
+    /**
+     * The record key as UTF-8 text, base64 when its bytes are not valid UTF-8
+     * (for example a key written by Java's `LongSerializer`), null when absent.
+     */
+    readonly key: string | null;
+    /**
+     * Every record header as text: the last value of a repeated header; a value
+     * that is not valid UTF-8 is left out.
+     */
+    readonly headers: EventHeaders;
+    /** The record value parsed as JSON. */
+    readonly payload: unknown;
+    /** Record timestamp in epoch milliseconds. */
+    readonly timestamp: string | undefined;
+}
+/** Options of {@link plainJsonDecoder}. Extractors returning undefined or `''` select the default. */
+interface PlainJsonDecoderOptions {
+    /**
+     * Event type handlers subscribe to: a fixed name for a topic that carries one
+     * kind of event, or an extractor such as `fromHeader('event-type')` or
+     * `fromPayloadField('type')`. Required; a record without a type is invalid.
+     */
+    readonly type: string | ((record: PlainJsonRecord) => string | undefined);
+    /**
+     * Event id the inbox deduplicates on, for example `fromHeader('message-id')`.
+     * Default, and fallback when the extractor finds none:
+     * `kafka:<topic>:<partition>:<offset>`, which is stable across redeliveries
+     * of the record. Prefer an id set by the producer when it may send the same
+     * event twice: two sends are two offsets, so only a producer id dedupes them.
+     */
+    readonly id?: (record: PlainJsonRecord) => string | undefined;
+    /** Ordering key. Default: the record key. When given, its result is final (undefined means no key). */
+    readonly key?: (record: PlainJsonRecord) => string | null | undefined;
+    /** Logical name of the producer. Default: the topic. */
+    readonly source?: string | ((record: PlainJsonRecord) => string | undefined);
+    /** When the event happened. Default: the record timestamp. */
+    readonly occurredAt?: (record: PlainJsonRecord) => Date | undefined;
+}
+/**
+ * Decodes a CloudEvents 1.0 record in either content mode of the Kafka
+ * protocol binding:
+ * - structured (`content-type: application/cloudevents+json`): the value is
+ *   a JSON object with `specversion`, `id`, `type`, `source` and optionally
+ *   `time`, `subject` (the key, else the record key), `data` (the payload,
+ *   null when absent) and a JSON `datacontenttype`. `data_base64` is refused,
+ *   because relaybus payloads are JSON. Extension attributes with string
+ *   values (`traceparent`, ...) become event headers, as do the record
+ *   headers; `correlationid` and `causationid` fill their envelope fields.
+ * - binary (anything else): exactly {@link decodeEnvelope}, the default decoder.
+ *
+ * @throws InvalidKafkaMessageError for a record that is neither.
+ */
+declare function decodeCloudEvent(record: DecodableMessage): EventEnvelope;
+/**
+ * Creates a decoder for records whose value is plain JSON, as written by a
+ * service that does not use relaybus (a plain kafkajs producer, Spring Kafka's
+ * `JsonSerializer`, a Go service): the value becomes the payload, the record
+ * headers become event headers, and the options say where the event type and
+ * id come from.
+ *
+ * A record that is a CloudEvent anyway (a `ce_specversion` header, or a
+ * structured `application/cloudevents+json` value) is decoded with
+ * {@link decodeCloudEvent} instead, so relaybus and foreign producers can share
+ * a topic. A missing or non-JSON value, a missing or invalid type, and an
+ * extractor returning anything but a string are invalid.
+ *
+ * ```ts
+ * new KafkaTransport({
+ *   kafka,
+ *   groupId: 'billing',
+ *   decode: plainJsonDecoder({ type: fromHeader('event-type'), id: fromHeader('message-id') }),
+ * });
+ * ```
+ *
+ * @throws ConfigurationError when a fixed `type` is not a valid event type.
+ */
+declare function plainJsonDecoder(options: PlainJsonDecoderOptions): KafkaMessageDecoder;
+
 /** A record the transport refused to hand to the bus because it is malformed. */
 interface InvalidKafkaMessage {
     readonly topic: string;
@@ -224,12 +343,25 @@ interface KafkaTransportOptions {
     readonly restart?: KafkaRestartOptions;
     /** Handling of records that cannot be processed for good. Default `'dead-letter'`. */
     readonly onInvalidMessage?: InvalidMessagePolicy;
+    /**
+     * How consumed records become events. Default: strict CloudEvents binary
+     * mode ({@link decodeEnvelope}), which is what relaybus producers write; any
+     * other record is malformed and goes to `onInvalidMessage`. Set it to read
+     * topics written by services that do not use relaybus:
+     * {@link decodeCloudEvent} also accepts structured mode, {@link plainJsonDecoder}
+     * maps plain JSON records, and a function of your own can do anything else
+     * (switch on `record.topic` to decode topics differently). Whatever a custom
+     * decoder throws marks the record as malformed, never crashes the consumer.
+     */
+    readonly decode?: KafkaMessageDecoder;
     /** Default `console`. */
     readonly logger?: RelayBusLogger;
 }
 /**
  * Kafka {@link Transport}. Events travel as CloudEvents binary-mode records
  * (`ce_*` headers, JSON value), so any CloudEvents-aware consumer can read them.
+ * Records from producers that do not use relaybus are read through the
+ * `decode` option.
  *
  * Publishing: idempotent producer, `acks=-1`, resolves after the broker ack and
  * rejects with {@link PublishTimeoutError} after `publishTimeout`. A timed-out
@@ -280,4 +412,33 @@ declare class KafkaTransport implements Transport {
     close(): Promise<void>;
 }
 
-export { type DecodableMessage, type EncodedMessage, HandlerFailedError, type InvalidKafkaMessage, InvalidKafkaMessageError, type InvalidMessagePolicy, type KafkaBytes, type KafkaClientLike, type KafkaConsumerConfig, type KafkaConsumerEvent, type KafkaConsumerEventNames, type KafkaConsumerLike, type KafkaEachMessagePayload, type KafkaIncomingHeaders, type KafkaIncomingMessage, type KafkaOutgoingHeaders, type KafkaOutgoingMessage, type KafkaProcessingOptions, type KafkaProducerConfig, type KafkaProducerLike, type KafkaProducerRecord, type KafkaRestartOptions, type KafkaRunConfig, KafkaTransport, type KafkaTransportOptions, PublishTimeoutError, UnencodableEventError, decodeEnvelope, encodeEnvelope };
+/**
+ * Reads one header of an incoming message, for the `type`, `id`, `key` or
+ * `source` option of {@link plainJsonDecoder}. Header names are compared
+ * exactly (Kafka header names are case-sensitive). Returns undefined when the
+ * header is absent, so the decoder falls back to its default.
+ *
+ * ```ts
+ * plainJsonDecoder({ type: fromHeader('event-type'), id: fromHeader('message-id') })
+ * ```
+ */
+declare function fromHeader(name: string): (message: {
+    readonly headers: EventHeaders;
+}) => string | undefined;
+/**
+ * Reads a field of the JSON payload by a dotted path (`'meta.eventType'`), for
+ * the `type`, `id`, `key` or `source` option of {@link plainJsonDecoder}. A
+ * string is returned as is and a safe integer as its decimal text (numeric ids
+ * are common). Anything else returns undefined so the decoder falls back to its
+ * default: a missing field, and also a fraction or an integer beyond 2^53, which
+ * JSON parsing may already have rounded, so two different ids could collide.
+ *
+ * ```ts
+ * plainJsonDecoder({ type: fromPayloadField('eventType'), key: fromPayloadField('order.id') })
+ * ```
+ */
+declare function fromPayloadField(path: string): (message: {
+    readonly payload: unknown;
+}) => string | undefined;
+
+export { type DecodableMessage, type DecodedEvent, type EncodedMessage, HandlerFailedError, type InvalidKafkaMessage, InvalidKafkaMessageError, type InvalidMessagePolicy, type KafkaBytes, type KafkaClientLike, type KafkaConsumerConfig, type KafkaConsumerEvent, type KafkaConsumerEventNames, type KafkaConsumerLike, type KafkaEachMessagePayload, type KafkaInboundRecord, type KafkaIncomingHeaders, type KafkaIncomingMessage, type KafkaMessageDecoder, type KafkaOutgoingHeaders, type KafkaOutgoingMessage, type KafkaProcessingOptions, type KafkaProducerConfig, type KafkaProducerLike, type KafkaProducerRecord, type KafkaRestartOptions, type KafkaRunConfig, KafkaTransport, type KafkaTransportOptions, type PlainJsonDecoderOptions, type PlainJsonRecord, PublishTimeoutError, UnencodableEventError, decodeCloudEvent, decodeEnvelope, encodeEnvelope, fromHeader, fromPayloadField, plainJsonDecoder };

package/dist/index.js

@@ -147,17 +147,20 @@ function resolveTime(text, recordTimestamp) {
   if (text !== void 0 && text.length > 0) {
     return parseTime(text);
   }
-  const milliseconds = Number(recordTimestamp);
-  const time = new Date(milliseconds);
-  if (recordTimestamp === void 0 || recordTimestamp === "" || Number.isNaN(time.getTime())) {
+  const time = readRecordTimestamp(recordTimestamp);
+  if (time === void 0) {
     throw invalid(`${HEADER.time} is missing and the record has no usable timestamp`);
   }
   return time;
 }
-function parseTime(text) {
+function readRecordTimestamp(recordTimestamp) {
+  const time = new Date(Number(recordTimestamp));
+  return recordTimestamp === void 0 || recordTimestamp === "" || Number.isNaN(time.getTime()) ? void 0 : time;
+}
+function parseTime(text, label = HEADER.time) {
   const time = new Date(text);
   if (!RFC3339_PATTERN.test(text) || Number.isNaN(time.getTime())) {
-    throw invalid(`${HEADER.time} is not an RFC 3339 timestamp: '${text}'`);
+    throw invalid(`${label} is not an RFC 3339 timestamp: '${text}'`);
   }
   return time;
 }
@@ -173,12 +176,15 @@ function parsePayload(value) {
   }
 }
 function resolveKey(subject, recordKey) {
-  const keyText = recordKey === null ? null : decodeText(recordKey, true, "Message key");
+  const keyText = readRecordKey(recordKey);
   if (subject !== void 0 && keyText !== null && subject !== keyText) {
     throw invalid(`${HEADER.subject} does not match the message key`);
   }
   return subject ?? keyText;
 }
+function readRecordKey(recordKey) {
+  return recordKey === null ? null : decodeText(recordKey, true, "Message key");
+}
 function collectUserHeaders(headers) {
   const entries = [];
   for (const [name, value] of headers) {
@@ -206,6 +212,219 @@ function toOutgoingHeaders(incoming) {
 function toBufferOrString(value) {
   return typeof value === "string" ? value : Buffer.from(value);
 }
+var STRUCTURED_MEDIA_TYPE = "application/cloudevents+json";
+var exactUtf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+var SPEC_VERSION2 = "1.0";
+var SPEC_VERSION_HEADER = "ce_specversion";
+var STRUCTURED_ATTRIBUTES = /* @__PURE__ */ new Set([
+  "specversion",
+  "id",
+  "type",
+  "source",
+  "time",
+  "subject",
+  "datacontenttype",
+  "dataschema",
+  "data",
+  "data_base64",
+  "correlationid",
+  "causationid"
+]);
+function decodeCloudEvent(record) {
+  const headers = readHeaders(record.headers);
+  if (mediaTypeOf(headers.get(CONTENT_TYPE)) !== STRUCTURED_MEDIA_TYPE) {
+    return decodeEnvelope(record);
+  }
+  const body = parsePayload(record.value);
+  if (!isJsonObject(body)) {
+    throw invalid("A structured CloudEvent must be a JSON object");
+  }
+  if (body["specversion"] !== SPEC_VERSION2) {
+    throw invalid(`specversion must be '${SPEC_VERSION2}'`);
+  }
+  const type = requiredAttribute(body, "type");
+  assertValidType(type, "type");
+  assertJsonDataContentType(optionalAttribute(body, "datacontenttype"));
+  if (Object.hasOwn(body, "data_base64")) {
+    throw invalid("data_base64 is not supported: relaybus payloads are JSON, send them as data");
+  }
+  const time = optionalAttribute(body, "time");
+  return {
+    id: requiredAttribute(body, "id"),
+    type,
+    source: requiredAttribute(body, "source"),
+    key: optionalAttribute(body, "subject") ?? readRecordKey(record.key),
+    payload: Object.hasOwn(body, "data") ? body["data"] : null,
+    headers: { ...collectUserHeaders(headers), ...extensionAttributes(body) },
+    correlationId: optionalAttribute(body, "correlationid") ?? null,
+    causationId: optionalAttribute(body, "causationid") ?? null,
+    occurredAt: time === void 0 ? requireRecordTimestamp(record) : parseTime(time, "time")
+  };
+}
+function plainJsonDecoder(options) {
+  if (typeof options.type === "string") {
+    assertEventType(options.type);
+  }
+  return (record) => {
+    const headers = readHeadersLeniently(record.headers);
+    if (record.headers?.[SPEC_VERSION_HEADER] !== void 0 || mediaTypeOf(headers[CONTENT_TYPE]) === STRUCTURED_MEDIA_TYPE) {
+      return decodeCloudEvent(record);
+    }
+    const plain = {
+      topic: record.topic,
+      partition: record.partition,
+      offset: record.offset,
+      key: readKeyLeniently(record.key),
+      headers,
+      payload: parsePayload(record.value),
+      timestamp: record.timestamp
+    };
+    const type = resolveText(options.type, plain, "type");
+    if (type === void 0) {
+      throw invalid("Record has no event type: the type option found none");
+    }
+    assertValidType(type, "The event type");
+    return {
+      id: resolveText(options.id, plain, "id") ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
+      type,
+      source: resolveText(options.source, plain, "source") ?? record.topic,
+      key: options.key === void 0 ? plain.key : resolveText(options.key, plain, "key") ?? null,
+      payload: plain.payload,
+      headers,
+      correlationId: null,
+      causationId: null,
+      occurredAt: resolveOccurredAt(options.occurredAt, plain, record)
+    };
+  };
+}
+function completeDecodedEvent(decoded, record) {
+  if (typeof decoded !== "object" || decoded === null) {
+    throw invalid("The decoder must return an event object synchronously");
+  }
+  if ("then" in decoded) {
+    observeRejection(decoded);
+    throw invalid("The decoder must return an event object synchronously");
+  }
+  const event = decoded;
+  return {
+    id: event.id,
+    type: event.type,
+    source: event.source ?? record.topic,
+    key: event.key === void 0 ? readRecordKey(record.key) : event.key,
+    payload: event.payload,
+    headers: event.headers ?? {},
+    correlationId: event.correlationId ?? null,
+    causationId: event.causationId ?? null,
+    occurredAt: event.occurredAt ?? requireRecordTimestamp(record)
+  };
+}
+function mediaTypeOf(contentType) {
+  return contentType?.split(";")[0]?.trim().toLowerCase();
+}
+function isJsonObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function optionalAttribute(body, name) {
+  const value = body[name];
+  if (value !== void 0 && typeof value !== "string") {
+    throw invalid(`Attribute '${name}' must be a string`);
+  }
+  return value;
+}
+function requiredAttribute(body, name) {
+  const value = optionalAttribute(body, name);
+  if (value === void 0 || value.length === 0) {
+    throw invalid(`Attribute '${name}' is missing or empty`);
+  }
+  return value;
+}
+function assertValidType(type, label) {
+  try {
+    assertEventType(type);
+  } catch (error) {
+    throw invalid(`${label} is not a valid event type: ${error.message}`);
+  }
+}
+function assertJsonDataContentType(dataContentType) {
+  const mediaType = mediaTypeOf(dataContentType);
+  if (mediaType !== void 0 && mediaType !== "application/json" && mediaType !== "text/json" && !mediaType.endsWith("+json")) {
+    throw invalid(`datacontenttype must be JSON, received '${dataContentType}'`);
+  }
+}
+function extensionAttributes(body) {
+  return Object.fromEntries(
+    Object.entries(body).filter(
+      (entry) => !STRUCTURED_ATTRIBUTES.has(entry[0]) && typeof entry[1] === "string"
+    )
+  );
+}
+function requireRecordTimestamp(record) {
+  const time = readRecordTimestamp(record.timestamp);
+  if (time === void 0) {
+    throw invalid("The event has no time and the record has no usable timestamp");
+  }
+  return time;
+}
+function readHeadersLeniently(incoming) {
+  const entries = [];
+  for (const [name, raw] of Object.entries(incoming ?? {})) {
+    const value = Array.isArray(raw) ? raw.at(-1) : raw;
+    const text = value === void 0 ? void 0 : decodeUtf8OrUndefined(value);
+    if (text !== void 0) {
+      entries.push([name, text]);
+    }
+  }
+  return Object.fromEntries(entries);
+}
+function decodeUtf8OrUndefined(value) {
+  if (typeof value === "string") {
+    return value;
+  }
+  try {
+    return exactUtf8.decode(value);
+  } catch {
+    return void 0;
+  }
+}
+function readKeyLeniently(key) {
+  if (key === null) {
+    return null;
+  }
+  try {
+    return readRecordKey(key);
+  } catch {
+    return Buffer.from(key).toString("base64");
+  }
+}
+function resolveText(option, record, name) {
+  const value = typeof option === "function" ? option(record) : option;
+  if (value === void 0 || value === null) {
+    return void 0;
+  }
+  if (typeof value !== "string") {
+    observeRejection(value);
+    throw invalid(`The ${name} option returned ${typeof value}, expected a string`);
+  }
+  return value.length === 0 ? void 0 : value;
+}
+function resolveOccurredAt(option, plain, record) {
+  const value = option?.(plain);
+  if (value === void 0) {
+    return requireRecordTimestamp(record);
+  }
+  if (!(value instanceof Date) || Number.isNaN(value.getTime())) {
+    observeRejection(value);
+    throw invalid("The occurredAt option returned no valid Date");
+  }
+  return value;
+}
+function observeRejection(value) {
+  if (typeof value === "object" && value !== null && "then" in value) {
+    Promise.resolve(value).catch(() => void 0);
+  }
+}
+
+// src/kafka-transport.ts
 var PublishTimeoutError = class extends RelayBusError {
   name = "PublishTimeoutError";
 };
@@ -233,6 +452,7 @@ var KafkaTransport = class {
   #producerConfig;
   #consumerConfig;
   #invalidPolicy;
+  #decode;
   #logger;
   #settings;
   #sleepers = /* @__PURE__ */ new Set();
@@ -260,6 +480,7 @@ var KafkaTransport = class {
     this.#producerConfig = options.producerConfig ?? {};
     this.#consumerConfig = options.consumerConfig ?? {};
     this.#invalidPolicy = options.onInvalidMessage ?? "dead-letter";
+    this.#decode = options.decode;
     this.#logger = options.logger ?? console;
     this.#settings = resolveSettings(options);
   }
@@ -441,16 +662,13 @@ var KafkaTransport = class {
   async #handleMessage(payload, onMessage) {
     let envelope;
     try {
-      envelope = decodeEnvelope(payload.message);
+      envelope = this.#decodeRecord(payload);
     } catch (error) {
-      if (!(error instanceof InvalidKafkaMessageError)) {
+      if (this.#decode === void 0 && !(error instanceof InvalidKafkaMessageError)) {
         throw toHandlerFailure(error);
       }
-      await this.#withRetries(
-        payload,
-        "handleInvalid",
-        () => this.#handleInvalid(payload, error.message)
-      );
+      const reason = toErrorInfo(error).message;
+      await this.#withRetries(payload, "handleInvalid", () => this.#handleInvalid(payload, reason));
       return;
     }
     await this.#withRetries(payload, "deliver", async () => {
@@ -464,6 +682,21 @@ var KafkaTransport = class {
       }
     });
   }
+  #decodeRecord({ topic, partition, message }) {
+    if (this.#decode === void 0) {
+      return decodeEnvelope(message);
+    }
+    const record = {
+      topic,
+      partition,
+      offset: message.offset,
+      key: message.key,
+      value: message.value,
+      headers: message.headers,
+      timestamp: message.timestamp
+    };
+    return completeDecodedEvent(this.#decode(record), record);
+  }
   async #handleInvalid(payload, reason) {
     const { topic, partition, message } = payload;
     const location = `topic=${topic} partition=${partition} offset=${message.offset}`;
@@ -705,7 +938,34 @@ function toHandlerFailure(error) {
   const { name, message } = toErrorInfo(error);
   return new HandlerFailedError(`${name}: ${message}`);
 }
+function fromHeader(name) {
+  if (name.length === 0) {
+    throw new ConfigurationError("fromHeader needs a header name");
+  }
+  return (message) => Object.hasOwn(message.headers, name) ? message.headers[name] : void 0;
+}
+function fromPayloadField(path) {
+  const segments = path.split(".");
+  if (segments.some((segment) => segment.length === 0)) {
+    throw new ConfigurationError(
+      `fromPayloadField needs a dotted path of non-empty names, received '${path}'`
+    );
+  }
+  return (message) => {
+    let value = message.payload;
+    for (const segment of segments) {
+      if (typeof value !== "object" || value === null || !Object.hasOwn(value, segment)) {
+        return void 0;
+      }
+      value = value[segment];
+    }
+    if (typeof value === "string") {
+      return value;
+    }
+    return Number.isSafeInteger(value) ? String(value) : void 0;
+  };
+}
 
-export { HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, UnencodableEventError, decodeEnvelope, encodeEnvelope };
+export { HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, UnencodableEventError, decodeCloudEvent, decodeEnvelope, encodeEnvelope, fromHeader, fromPayloadField, plainJsonDecoder };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map