@relaybus/kafka 0.0.0-stage → 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,283 @@
+import { RelayBusError, NonRetryableError, EventEnvelope, Duration, Transport, RelayBusLogger, TransportMessageHandler } from '@relaybus/core';
+
+/**
+ * Structural views of the Kafka client API used by this package.
+ *
+ * They are satisfied by both `kafkajs` (`new Kafka(...)`) and
+ * `@confluentinc/kafka-javascript` (`new KafkaJS.Kafka(...)`, the KafkaJS
+ * compatible API), so neither is a hard dependency. Only the members listed
+ * here are ever called. Methods are declared with method syntax on purpose:
+ * it keeps the real client classes assignable despite their richer signatures.
+ */
+/** Bytes as delivered by a consumer: `Buffer` in both supported clients. */
+type KafkaBytes = Uint8Array;
+/** Header values a consumer may hand over: bytes, a string, a list of either, or nothing. */
+type KafkaIncomingHeaders = Readonly<Record<string, KafkaBytes | string | readonly (KafkaBytes | string)[] | undefined>>;
+/** Header values accepted when producing. */
+type KafkaOutgoingHeaders = Readonly<Record<string, Buffer | string | (Buffer | string)[]>>;
+/** One record handed to the producer. */
+interface KafkaOutgoingMessage {
+    readonly key: string | Buffer | null;
+    readonly value: string | Buffer | null;
+    readonly headers: KafkaOutgoingHeaders;
+}
+/** Argument of `producer.send`. */
+interface KafkaProducerRecord {
+    readonly topic: string;
+    readonly messages: KafkaOutgoingMessage[];
+    /** Always `-1`: wait for all in-sync replicas. */
+    readonly acks: number;
+}
+/** Producer settings this package decides itself; user settings are merged below them. */
+interface KafkaProducerConfig {
+    readonly idempotent: boolean;
+    readonly maxInFlightRequests: number;
+}
+/** Consumer settings this package decides itself; user settings are merged below them. */
+interface KafkaConsumerConfig {
+    readonly groupId: string;
+}
+/** Subset of the producer API: `connect`, `send`, `disconnect`. */
+interface KafkaProducerLike {
+    connect(): Promise<void>;
+    send(record: KafkaProducerRecord): Promise<unknown>;
+    disconnect(): Promise<void>;
+}
+/** A message as delivered to `eachMessage`. */
+interface KafkaIncomingMessage {
+    readonly key: KafkaBytes | null;
+    readonly value: KafkaBytes | null;
+    readonly headers?: KafkaIncomingHeaders | undefined;
+    readonly offset: string;
+    /** Record timestamp as epoch milliseconds in a string (kafkajs and Confluent). */
+    readonly timestamp?: string | undefined;
+}
+/** Argument of the `eachMessage` callback. `pause` and others are not used. */
+interface KafkaEachMessagePayload {
+    readonly topic: string;
+    readonly partition: number;
+    readonly message: KafkaIncomingMessage;
+    heartbeat(): Promise<void>;
+}
+/** Argument of `consumer.run`. Offsets are committed automatically after `eachMessage` resolves. */
+interface KafkaRunConfig {
+    eachMessage(payload: KafkaEachMessagePayload): Promise<void>;
+}
+/**
+ * Instrumentation event names exposed as `consumer.events`. Optional: a client
+ * without them still works, but start-up cannot wait for the group join and
+ * `isConsuming()` cannot observe crashes.
+ */
+interface KafkaConsumerEventNames {
+    readonly GROUP_JOIN?: string;
+    readonly CRASH?: string;
+    readonly REBALANCING?: string;
+    readonly STOP?: string;
+    readonly DISCONNECT?: string;
+}
+/** Instrumentation event; `payload` carries `error` and `restart` for a crash. */
+interface KafkaConsumerEvent {
+    readonly payload?: unknown;
+}
+/** Subset of the consumer API: `connect`, `subscribe`, `run`, `disconnect`, optional `on`/`events`. */
+interface KafkaConsumerLike {
+    readonly events?: KafkaConsumerEventNames | undefined;
+    on?(eventName: string, listener: (event: KafkaConsumerEvent) => void): unknown;
+    connect(): Promise<void>;
+    subscribe(subscription: {
+        topics: string[];
+        fromBeginning: boolean;
+    }): Promise<void>;
+    run(config: KafkaRunConfig): Promise<void>;
+    disconnect(): Promise<void>;
+}
+/** Subset of the client API: the `producer` and `consumer` factories. */
+interface KafkaClientLike {
+    producer(config: KafkaProducerConfig): KafkaProducerLike;
+    consumer(config: KafkaConsumerConfig): KafkaConsumerLike;
+}
+
+/** A Kafka record that cannot be turned into an {@link EventEnvelope}. Never retried. */
+declare class InvalidKafkaMessageError extends RelayBusError {
+}
+/**
+ * An event that cannot be written to Kafka: its payload is not representable as
+ * JSON (`undefined`, a function, a symbol, a `BigInt`, a circular structure) or
+ * its `occurredAt` is not an RFC 3339 instant (invalid date, year outside 0..9999).
+ * Retrying cannot help, so it is a {@link NonRetryableError}.
+ */
+declare class UnencodableEventError extends NonRetryableError {
+    name: string;
+}
+/** The wire form of one event: a Kafka record in CloudEvents binary content mode. */
+interface EncodedMessage {
+    readonly key: string | null;
+    readonly value: string;
+    readonly headers: Record<string, string>;
+}
+/** The parts of an incoming Kafka record the decoder reads. */
+interface DecodableMessage {
+    readonly key: KafkaBytes | null;
+    readonly value: KafkaBytes | null;
+    readonly headers?: KafkaIncomingHeaders | undefined;
+    /** Record timestamp in epoch milliseconds, used when `ce_time` is absent. */
+    readonly timestamp?: string | undefined;
+}
+/**
+ * Encodes an envelope as a CloudEvents Kafka binary-mode record: the value is
+ * the JSON payload, the event attributes travel as `ce_*` headers.
+ *
+ * User headers that would collide with protocol names (`ce_*`, `content-type`,
+ * `x-relaybus-*`) are stored under an `x-relaybus-h-` prefix and restored on
+ * decode, so every header survives a round trip unchanged.
+ */
+declare function encodeEnvelope(envelope: EventEnvelope): EncodedMessage;
+/**
+ * Decodes a Kafka record strictly: the CloudEvents attributes `ce_specversion`
+ * (`1.0`), `ce_id`, `ce_type`, `ce_source`, a JSON `content-type` and a JSON
+ * value are required. `ce_time` is optional in CloudEvents: when absent, the
+ * Kafka record timestamp is used; when present it must be RFC 3339.
+ *
+ * @throws InvalidKafkaMessageError for anything else. It is the only error
+ * thrown for malformed input, so callers can tell poison messages from bugs.
+ */
+declare function decodeEnvelope(message: DecodableMessage): EventEnvelope;
+
+/** A record the transport refused to hand to the bus because it is malformed. */
+interface InvalidKafkaMessage {
+    readonly topic: string;
+    readonly partition: number;
+    readonly offset: string;
+    /** Why the record was rejected. */
+    readonly reason: string;
+    readonly key: Uint8Array | null;
+    readonly value: Uint8Array | null;
+}
+/**
+ * What to do with a record that cannot be processed for good (it cannot be
+ * decoded, or the bus rejected it with a `NonRetryableError`):
+ * - `'dead-letter'` (default): the original record is produced to
+ *   `<source topic>.dead` with `relaybus-invalid-reason`, `relaybus-source-topic`,
+ *   `relaybus-source-partition` and `relaybus-source-offset` headers added to the
+ *   original ones; the offset advances only after the broker acked the copy.
+ * - `{ deadLetterTopic }`: same, with a fixed topic name or a function of the
+ *   source topic.
+ * - a function: called with the record; if it rejects, the record is retried
+ *   like any other failure, so nothing is dropped unless the function returns.
+ * - `'skip'`: log an error and drop the record. Explicit opt-in only.
+ *
+ * The dead-letter topic must exist or the broker must auto-create topics;
+ * otherwise the send fails and the record stays uncommitted (nothing is lost).
+ */
+type InvalidMessagePolicy = 'dead-letter' | 'skip' | ((message: InvalidKafkaMessage) => void | Promise<void>) | {
+    readonly deadLetterTopic: string | ((sourceTopic: string) => string);
+};
+/** `publish` did not get a broker ack within `publishTimeout`. The outbox retries it. */
+declare class PublishTimeoutError extends RelayBusError {
+    name: string;
+}
+/** A handler failure re-thrown to the Kafka client, stripped of anything that could make it fatal. */
+declare class HandlerFailedError extends RelayBusError {
+    name: string;
+}
+/** Bounded in-process retries around the inbound handler (the bus's `receive`). */
+interface KafkaProcessingOptions {
+    /** Attempts per delivery of a record before the error is handed to the client. Default 5. */
+    readonly maxAttempts?: number;
+    /** Delay before the second attempt; doubles each attempt. Default `'200ms'`. */
+    readonly initialBackoff?: Duration;
+    /** Upper bound of the delay between attempts. Default `'5s'`. */
+    readonly maxBackoff?: Duration;
+    /** Interval of the heartbeats sent while sleeping between attempts. Default `'1s'`. */
+    readonly heartbeatInterval?: Duration;
+}
+/** Backoff of the consumer restart loop that runs after an unrecoverable crash. */
+interface KafkaRestartOptions {
+    /** Delay before the second restart attempt; doubles each failure. Default `'1s'`. */
+    readonly initialBackoff?: Duration;
+    /** Upper bound of the delay between restart attempts. Default `'30s'`. */
+    readonly maxBackoff?: Duration;
+}
+interface KafkaTransportOptions {
+    /** A `kafkajs` `Kafka` or `@confluentinc/kafka-javascript` `KafkaJS.Kafka` instance. */
+    readonly kafka: KafkaClientLike;
+    /** Name referenced by routes and subscriptions. Default `'kafka'`. */
+    readonly name?: string;
+    /** Consumer group id. Required as soon as `subscribe` is used. */
+    readonly groupId?: string;
+    /**
+     * Where a group without committed offsets starts. Default `true` (earliest),
+     * so events published before the first start are not skipped.
+     */
+    readonly fromBeginning?: boolean;
+    /** Producer `maxInFlightRequests`, 1 to 5. Default 1, which keeps retries from reordering a key. */
+    readonly maxInFlightRequests?: number;
+    /** Extra producer settings; `idempotent` and `maxInFlightRequests` always win. */
+    readonly producerConfig?: object;
+    /** Extra consumer settings; `groupId` always wins. */
+    readonly consumerConfig?: object;
+    readonly processing?: KafkaProcessingOptions;
+    /** How long `subscribe` waits for the consumer group join before failing. Default `'30s'`. */
+    readonly startupTimeout?: Duration;
+    /** How long `publish` waits for the broker ack before rejecting. Default `'30s'`. */
+    readonly publishTimeout?: Duration;
+    readonly restart?: KafkaRestartOptions;
+    /** Handling of records that cannot be processed for good. Default `'dead-letter'`. */
+    readonly onInvalidMessage?: InvalidMessagePolicy;
+    /** Default `console`. */
+    readonly logger?: RelayBusLogger;
+}
+/**
+ * Kafka {@link Transport}. Events travel as CloudEvents binary-mode records
+ * (`ce_*` headers, JSON value), so any CloudEvents-aware consumer can read them.
+ *
+ * Publishing: idempotent producer, `acks=-1`, resolves after the broker ack and
+ * rejects with {@link PublishTimeoutError} after `publishTimeout`. A timed-out
+ * send may still complete later; the outbox then retries it and the duplicate is
+ * absorbed by the consumer's inbox dedup. The record key is the envelope key, so events sharing a key land on one
+ * partition and keep their order; events without a key have no ordering.
+ *
+ * Subscribing: one consumer, one group, one `subscribe` call. `eachMessage`
+ * awaits the handler; the client commits the offset only after it resolved. A
+ * failing handler is retried in-process (bounded, with backoff) and then the
+ * error is rethrown (as a {@link HandlerFailedError}, so the client never treats
+ * it as fatal) and the client does not commit the offset and redelivers the
+ * record. A record is therefore never skipped because of a handler failure; the
+ * partition simply waits. Records that can never succeed (undecodable, or
+ * rejected with a `NonRetryableError`) are dead-lettered instead. If the client
+ * reports an unrecoverable crash anyway, the consumer run loop is restarted with
+ * bounded backoff until the transport is closed.
+ */
+declare class KafkaTransport implements Transport {
+    #private;
+    readonly name: string;
+    constructor(options: KafkaTransportOptions);
+    /**
+     * True while the consumer is joined to its group, not between a crash or
+     * rebalance and the next join, and no partition is stalled on a record whose
+     * handler keeps failing. Use it for readiness so a consumer that silently
+     * stopped is noticed.
+     */
+    isConsuming(): boolean;
+    /**
+     * True once the producer is connected and, if the transport subscribed, the
+     * consumer is consuming. False before `connect`, after `close` and while a
+     * dead consumer is being restarted.
+     */
+    isReady(): boolean;
+    /** Connects the producer. Called by the bus on start; `publish` also connects on demand. */
+    connect(): Promise<void>;
+    publish(envelope: EventEnvelope, destination: string): Promise<void>;
+    /**
+     * Starts the consumer and resolves once it joined its group, so a start-up
+     * that cannot consume fails loudly instead of looking healthy.
+     *
+     * @throws ConfigurationError when `groupId` is missing, `destinations` is empty
+     * or the transport already subscribed (create one transport per subscription).
+     */
+    subscribe(destinations: readonly string[], onMessage: TransportMessageHandler): Promise<void>;
+    /** Stops consuming and disconnects the consumer and the producer. Safe to call twice. */
+    close(): Promise<void>;
+}
+
+export { type DecodableMessage, type EncodedMessage, HandlerFailedError, type InvalidKafkaMessage, InvalidKafkaMessageError, type InvalidMessagePolicy, type KafkaBytes, type KafkaClientLike, type KafkaConsumerConfig, type KafkaConsumerEvent, type KafkaConsumerEventNames, type KafkaConsumerLike, type KafkaEachMessagePayload, type KafkaIncomingHeaders, type KafkaIncomingMessage, type KafkaOutgoingHeaders, type KafkaOutgoingMessage, type KafkaProcessingOptions, type KafkaProducerConfig, type KafkaProducerLike, type KafkaProducerRecord, type KafkaRestartOptions, type KafkaRunConfig, KafkaTransport, type KafkaTransportOptions, PublishTimeoutError, UnencodableEventError, decodeEnvelope, encodeEnvelope };