@drarzter/kafka-client 0.3.0 → 0.5.0

package/dist/envelope-QK1trQu4.d.ts

@@ -0,0 +1,319 @@
+/**
+ * Any validation library with a `.parse()` method.
+ * Works with Zod, Valibot, ArkType, or any custom validator.
+ *
+ * @example
+ * ```ts
+ * import { z } from 'zod';
+ * const schema: SchemaLike<{ id: string }> = z.object({ id: z.string() });
+ * ```
+ */
+interface SchemaLike<T = any> {
+    parse(data: unknown): T;
+}
+/** Infer the output type from a SchemaLike. */
+type InferSchema<S extends SchemaLike> = S extends SchemaLike<infer T> ? T : never;
+/**
+ * A typed topic descriptor that pairs a topic name with its message type.
+ * Created via the `topic()` factory function.
+ *
+ * @typeParam N - The literal topic name string.
+ * @typeParam M - The message payload type for this topic.
+ */
+interface TopicDescriptor<N extends string = string, M extends Record<string, any> = Record<string, any>> {
+    readonly __topic: N;
+    /** @internal Phantom type — never has a real value at runtime. */
+    readonly __type: M;
+    /** Runtime schema validator. Present only when created via `topic().schema()`. */
+    readonly __schema?: SchemaLike<M>;
+}
+/**
+ * Define a typed topic descriptor.
+ *
+ * @example
+ * ```ts
+ * // Without schema — type provided explicitly:
+ * const OrderCreated = topic('order.created')<{ orderId: string; amount: number }>();
+ *
+ * // With schema — type inferred from schema:
+ * const OrderCreated = topic('order.created').schema(z.object({
+ *   orderId: z.string(),
+ *   amount: z.number(),
+ * }));
+ *
+ * // Use with KafkaClient:
+ * await kafka.sendMessage(OrderCreated, { orderId: '123', amount: 100 });
+ *
+ * // Use with @SubscribeTo:
+ * @SubscribeTo(OrderCreated)
+ * async handleOrder(msg) { ... }
+ * ```
+ */
+declare function topic<N extends string>(name: N): {
+    <M extends Record<string, any>>(): TopicDescriptor<N, M>;
+    schema<S extends SchemaLike<Record<string, any>>>(schema: S): TopicDescriptor<N, InferSchema<S>>;
+};
+/**
+ * Build a topic-message map type from a union of TopicDescriptors.
+ *
+ * @example
+ * ```ts
+ * const OrderCreated = topic('order.created')<{ orderId: string }>();
+ * const OrderCompleted = topic('order.completed')<{ completedAt: string }>();
+ *
+ * type MyTopics = TopicsFrom<typeof OrderCreated | typeof OrderCompleted>;
+ * // { 'order.created': { orderId: string }; 'order.completed': { completedAt: string } }
+ * ```
+ */
+type TopicsFrom<D extends TopicDescriptor<any, any>> = {
+    [K in D as K["__topic"]]: K["__type"];
+};
+
+/**
+ * Mapping of topic names to their message types.
+ * Define this interface to get type-safe publish/subscribe across your app.
+ *
+ * @example
+ * ```ts
+ * // with explicit extends (IDE hints for values)
+ * interface MyTopics extends TTopicMessageMap {
+ *   "orders.created": { orderId: string; amount: number };
+ *   "users.updated": { userId: string; name: string };
+ * }
+ *
+ * // or plain interface / type — works the same
+ * interface MyTopics {
+ *   "orders.created": { orderId: string; amount: number };
+ * }
+ * ```
+ */
+type TTopicMessageMap = {
+    [topic: string]: Record<string, any>;
+};
+/**
+ * Generic constraint for topic-message maps.
+ * Works with both `type` aliases and `interface` declarations.
+ */
+type TopicMapConstraint<T> = {
+    [K in keyof T]: Record<string, any>;
+};
+type ClientId = string;
+type GroupId = string;
+type MessageHeaders = Record<string, string>;
+/** Options for sending a single message. */
+interface SendOptions {
+    /** Partition key for message routing. */
+    key?: string;
+    /** Custom headers attached to the message (merged with auto-generated envelope headers). */
+    headers?: MessageHeaders;
+    /** Override the auto-propagated correlation ID (default: inherited from ALS context or new UUID). */
+    correlationId?: string;
+    /** Schema version for the payload. Default: `1`. */
+    schemaVersion?: number;
+    /** Override the auto-generated event ID (UUID v4). */
+    eventId?: string;
+}
+/** Shape of each item in a `sendBatch` call. */
+interface BatchMessageItem<V> {
+    value: V;
+    key?: string;
+    headers?: MessageHeaders;
+    correlationId?: string;
+    schemaVersion?: number;
+    eventId?: string;
+}
+/** Metadata exposed to batch consumer handlers. */
+interface BatchMeta {
+    /** Partition number for this batch. */
+    partition: number;
+    /** Highest offset available on the broker for this partition. */
+    highWatermark: string;
+    /** Send a heartbeat to the broker to prevent session timeout. */
+    heartbeat(): Promise<void>;
+    /** Mark an offset as processed (for manual offset management). */
+    resolveOffset(offset: string): void;
+    /** Commit offsets if the auto-commit threshold has been reached. */
+    commitOffsetsIfNecessary(): Promise<void>;
+}
+/** Options for configuring a Kafka consumer. */
+interface ConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> {
+    /** Override the default consumer group ID from the constructor. */
+    groupId?: string;
+    /** Start reading from earliest offset. Default: `false`. */
+    fromBeginning?: boolean;
+    /** Automatically commit offsets. Default: `true`. */
+    autoCommit?: boolean;
+    /** Retry policy for failed message processing. */
+    retry?: RetryOptions;
+    /** Send failed messages to a Dead Letter Queue (`<topic>.dlq`). */
+    dlq?: boolean;
+    /** Interceptors called before/after each message. */
+    interceptors?: ConsumerInterceptor<T>[];
+    /** @internal Schema map populated by @SubscribeTo when descriptors have schemas. */
+    schemas?: Map<string, SchemaLike>;
+    /** Retry config for `consumer.subscribe()` when the topic doesn't exist yet. */
+    subscribeRetry?: SubscribeRetryOptions;
+}
+/** Configuration for consumer retry behavior. */
+interface RetryOptions {
+    /** Maximum number of retry attempts before giving up. */
+    maxRetries: number;
+    /** Base delay between retries in ms (multiplied by attempt number). Default: `1000`. */
+    backoffMs?: number;
+}
+/**
+ * Interceptor hooks for consumer message processing.
+ * All methods are optional — implement only what you need.
+ *
+ * Interceptors are per-consumer. For client-wide hooks (e.g. OTel),
+ * use `KafkaInstrumentation` instead.
+ */
+interface ConsumerInterceptor<T extends TopicMapConstraint<T> = TTopicMessageMap> {
+    /** Called before the message handler. */
+    before?(envelope: EventEnvelope<T[keyof T]>): Promise<void> | void;
+    /** Called after the message handler succeeds. */
+    after?(envelope: EventEnvelope<T[keyof T]>): Promise<void> | void;
+    /** Called when the message handler throws. */
+    onError?(envelope: EventEnvelope<T[keyof T]>, error: Error): Promise<void> | void;
+}
+/**
+ * Client-wide instrumentation hooks for both send and consume paths.
+ * Use this for cross-cutting concerns like tracing and metrics.
+ *
+ * @see `otelInstrumentation()` from `@drarzter/kafka-client/otel`
+ */
+interface KafkaInstrumentation {
+    /** Called before sending — can mutate `headers` (e.g. inject `traceparent`). */
+    beforeSend?(topic: string, headers: MessageHeaders): void;
+    /** Called after a successful send. */
+    afterSend?(topic: string): void;
+    /** Called before the consumer handler. Return a cleanup function called after the handler. */
+    beforeConsume?(envelope: EventEnvelope<any>): (() => void) | void;
+    /** Called when the consumer handler throws. */
+    onConsumeError?(envelope: EventEnvelope<any>, error: Error): void;
+}
+/** Context passed to the `transaction()` callback with type-safe send methods. */
+interface TransactionContext<T extends TopicMapConstraint<T>> {
+    send<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
+    send<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, message: D["__type"], options?: SendOptions): Promise<void>;
+    sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>): Promise<void>;
+    sendBatch<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, messages: Array<BatchMessageItem<D["__type"]>>): Promise<void>;
+}
+/** Interface describing all public methods of the Kafka client. */
+interface IKafkaClient<T extends TopicMapConstraint<T>> {
+    checkStatus(): Promise<{
+        topics: string[];
+    }>;
+    startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (envelope: EventEnvelope<T[K[number]]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
+    startConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleMessage: (envelope: EventEnvelope<D["__type"]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
+    startBatchConsumer<K extends Array<keyof T>>(topics: K, handleBatch: (envelopes: EventEnvelope<T[K[number]]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
+    startBatchConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleBatch: (envelopes: EventEnvelope<D["__type"]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
+    stopConsumer(): Promise<void>;
+    sendMessage<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
+    sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>): Promise<void>;
+    transaction(fn: (ctx: TransactionContext<T>) => Promise<void>): Promise<void>;
+    getClientId: () => ClientId;
+    disconnect(): Promise<void>;
+}
+/**
+ * Logger interface for KafkaClient.
+ * Compatible with NestJS Logger, console, winston, pino, or any custom logger.
+ */
+interface KafkaLogger {
+    log(message: string): void;
+    warn(message: string, ...args: any[]): void;
+    error(message: string, ...args: any[]): void;
+}
+/** Options for `KafkaClient` constructor. */
+interface KafkaClientOptions {
+    /** Auto-create topics via admin before the first `sendMessage`, `sendBatch`, or `transaction` for each topic. Useful for development — not recommended in production. */
+    autoCreateTopics?: boolean;
+    /** When `true`, string topic keys are validated against any schema previously registered via a TopicDescriptor. Default: `true`. */
+    strictSchemas?: boolean;
+    /** Custom logger. Defaults to console with `[KafkaClient:<clientId>]` prefix. */
+    logger?: KafkaLogger;
+    /** Number of partitions for auto-created topics. Default: `1`. */
+    numPartitions?: number;
+    /** Client-wide instrumentation hooks (e.g. OTel). Applied to both send and consume paths. */
+    instrumentation?: KafkaInstrumentation[];
+}
+/** Options for consumer subscribe retry when topic doesn't exist yet. */
+interface SubscribeRetryOptions {
+    /** Maximum number of subscribe attempts. Default: `5`. */
+    retries?: number;
+    /** Delay between retries in ms. Default: `5000`. */
+    backoffMs?: number;
+}
+
+declare const HEADER_EVENT_ID = "x-event-id";
+declare const HEADER_CORRELATION_ID = "x-correlation-id";
+declare const HEADER_TIMESTAMP = "x-timestamp";
+declare const HEADER_SCHEMA_VERSION = "x-schema-version";
+declare const HEADER_TRACEPARENT = "traceparent";
+/**
+ * Typed wrapper combining a parsed message payload with Kafka metadata
+ * and envelope headers.
+ *
+ * On **send**, the library auto-generates envelope headers
+ * (`x-event-id`, `x-correlation-id`, `x-timestamp`, `x-schema-version`).
+ *
+ * On **consume**, the library extracts those headers and assembles
+ * an `EventEnvelope` that is passed to the handler.
+ */
+interface EventEnvelope<T> {
+    /** Deserialized + validated message body. */
+    payload: T;
+    /** Topic the message was produced to / consumed from. */
+    topic: string;
+    /** Kafka partition (consume-side only, `-1` on send). */
+    partition: number;
+    /** Kafka offset (consume-side only, empty string on send). */
+    offset: string;
+    /** ISO-8601 timestamp set by the producer. */
+    timestamp: string;
+    /** Unique ID for this event (UUID v4). */
+    eventId: string;
+    /** Correlation ID — auto-propagated via AsyncLocalStorage. */
+    correlationId: string;
+    /** Schema version of the payload. */
+    schemaVersion: number;
+    /** W3C Trace Context `traceparent` header (set by OTel instrumentation). */
+    traceparent?: string;
+    /** All decoded Kafka headers for extensibility. */
+    headers: MessageHeaders;
+}
+interface EnvelopeCtx {
+    correlationId: string;
+    traceparent?: string;
+}
+/** Read the current envelope context (correlationId / traceparent) from ALS. */
+declare function getEnvelopeContext(): EnvelopeCtx | undefined;
+/** Execute `fn` inside an envelope context so nested sends inherit correlationId. */
+declare function runWithEnvelopeContext<R>(ctx: EnvelopeCtx, fn: () => R): R;
+/** Options accepted by `buildEnvelopeHeaders`. */
+interface EnvelopeHeaderOptions {
+    correlationId?: string;
+    schemaVersion?: number;
+    eventId?: string;
+    headers?: MessageHeaders;
+}
+/**
+ * Generate envelope headers for the send path.
+ *
+ * Priority for `correlationId`:
+ *   explicit option → ALS context → new UUID.
+ */
+declare function buildEnvelopeHeaders(options?: EnvelopeHeaderOptions): MessageHeaders;
+/**
+ * Decode kafkajs headers (`Record<string, Buffer | string | undefined>`)
+ * into plain `Record<string, string>`.
+ */
+declare function decodeHeaders(raw: Record<string, Buffer | string | (Buffer | string)[] | undefined> | undefined): MessageHeaders;
+/**
+ * Build an `EventEnvelope` from a consumed kafkajs message.
+ * Tolerates missing envelope headers — generates defaults so messages
+ * from non-envelope producers still work.
+ */
+declare function extractEnvelope<T>(payload: T, headers: MessageHeaders, topic: string, partition: number, offset: string): EventEnvelope<T>;
+
+export { type BatchMessageItem as B, type ClientId as C, type EnvelopeHeaderOptions as E, type GroupId as G, HEADER_CORRELATION_ID as H, type IKafkaClient as I, type KafkaInstrumentation as K, type MessageHeaders as M, type RetryOptions as R, type SchemaLike as S, type TopicMapConstraint as T, type ConsumerOptions as a, type TopicDescriptor as b, type BatchMeta as c, type ConsumerInterceptor as d, type EventEnvelope as e, HEADER_EVENT_ID as f, HEADER_SCHEMA_VERSION as g, HEADER_TIMESTAMP as h, HEADER_TRACEPARENT as i, type InferSchema as j, type KafkaClientOptions as k, type KafkaLogger as l, type SendOptions as m, type SubscribeRetryOptions as n, type TTopicMessageMap as o, type TopicsFrom as p, type TransactionContext as q, buildEnvelopeHeaders as r, decodeHeaders as s, extractEnvelope as t, getEnvelopeContext as u, runWithEnvelopeContext as v, topic as w };

package/dist/index.d.mts

@@ -1,5 +1,7 @@
-import { TopicMapConstraint, ClientId, GroupId, SchemaLike, ConsumerOptions, TopicDescriptor, KafkaClient } from './core.mjs';
-export { BatchMeta, ConsumerInterceptor, IKafkaClient, InferSchema, KafkaClientOptions, KafkaLogger, KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError, MessageHeaders, RetryOptions, SendOptions, SubscribeRetryOptions, TTopicMessageMap, TopicsFrom, TransactionContext, topic } from './core.mjs';
+import { KafkaClient } from './core.mjs';
+export { KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError } from './core.mjs';
+import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, S as SchemaLike, a as ConsumerOptions, b as TopicDescriptor } from './envelope-QK1trQu4.mjs';
+export { B as BatchMessageItem, c as BatchMeta, d as ConsumerInterceptor, E as EnvelopeHeaderOptions, e as EventEnvelope, H as HEADER_CORRELATION_ID, f as HEADER_EVENT_ID, g as HEADER_SCHEMA_VERSION, h as HEADER_TIMESTAMP, i as HEADER_TRACEPARENT, I as IKafkaClient, j as InferSchema, k as KafkaClientOptions, l as KafkaLogger, M as MessageHeaders, R as RetryOptions, m as SendOptions, n as SubscribeRetryOptions, o as TTopicMessageMap, p as TopicsFrom, q as TransactionContext, r as buildEnvelopeHeaders, s as decodeHeaders, t as extractEnvelope, u as getEnvelopeContext, v as runWithEnvelopeContext, w as topic } from './envelope-QK1trQu4.mjs';
 import { DynamicModule, OnModuleInit } from '@nestjs/common';
 import { DiscoveryService, ModuleRef } from '@nestjs/core';
 
@@ -17,6 +19,12 @@ interface KafkaModuleOptions {
     isGlobal?: boolean;
     /** Auto-create topics via admin on first use (send/consume). Useful for development. */
     autoCreateTopics?: boolean;
+    /** When `true`, string topic keys are validated against any schema previously registered via a TopicDescriptor. Default: `true`. */
+    strictSchemas?: boolean;
+    /** Number of partitions for auto-created topics. Default: `1`. */
+    numPartitions?: number;
+    /** Client-wide instrumentation hooks (e.g. OTel). Applied to both send and consume paths. */
+    instrumentation?: KafkaInstrumentation[];
 }
 /** Async configuration for `KafkaModule.registerAsync()` with dependency injection. */
 interface KafkaModuleAsyncOptions {
@@ -86,4 +94,4 @@ declare class KafkaHealthIndicator {
     check<T extends TopicMapConstraint<T>>(client: KafkaClient<T>): Promise<KafkaHealthResult>;
 }
 
-export { ClientId, ConsumerOptions, GroupId, InjectKafkaClient, KAFKA_CLIENT, KAFKA_SUBSCRIBER_METADATA, KafkaClient, KafkaExplorer, KafkaHealthIndicator, type KafkaHealthResult, KafkaModule, type KafkaModuleAsyncOptions, type KafkaModuleOptions, type KafkaSubscriberMetadata, SchemaLike, SubscribeTo, TopicDescriptor, TopicMapConstraint, getKafkaClientToken };
+export { ClientId, ConsumerOptions, GroupId, InjectKafkaClient, KAFKA_CLIENT, KAFKA_SUBSCRIBER_METADATA, KafkaClient, KafkaExplorer, KafkaHealthIndicator, type KafkaHealthResult, KafkaInstrumentation, KafkaModule, type KafkaModuleAsyncOptions, type KafkaModuleOptions, type KafkaSubscriberMetadata, SchemaLike, SubscribeTo, TopicDescriptor, TopicMapConstraint, getKafkaClientToken };

package/dist/index.d.ts

@@ -1,5 +1,7 @@
-import { TopicMapConstraint, ClientId, GroupId, SchemaLike, ConsumerOptions, TopicDescriptor, KafkaClient } from './core.js';
-export { BatchMeta, ConsumerInterceptor, IKafkaClient, InferSchema, KafkaClientOptions, KafkaLogger, KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError, MessageHeaders, RetryOptions, SendOptions, SubscribeRetryOptions, TTopicMessageMap, TopicsFrom, TransactionContext, topic } from './core.js';
+import { KafkaClient } from './core.js';
+export { KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError } from './core.js';
+import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, S as SchemaLike, a as ConsumerOptions, b as TopicDescriptor } from './envelope-QK1trQu4.js';
+export { B as BatchMessageItem, c as BatchMeta, d as ConsumerInterceptor, E as EnvelopeHeaderOptions, e as EventEnvelope, H as HEADER_CORRELATION_ID, f as HEADER_EVENT_ID, g as HEADER_SCHEMA_VERSION, h as HEADER_TIMESTAMP, i as HEADER_TRACEPARENT, I as IKafkaClient, j as InferSchema, k as KafkaClientOptions, l as KafkaLogger, M as MessageHeaders, R as RetryOptions, m as SendOptions, n as SubscribeRetryOptions, o as TTopicMessageMap, p as TopicsFrom, q as TransactionContext, r as buildEnvelopeHeaders, s as decodeHeaders, t as extractEnvelope, u as getEnvelopeContext, v as runWithEnvelopeContext, w as topic } from './envelope-QK1trQu4.js';
 import { DynamicModule, OnModuleInit } from '@nestjs/common';
 import { DiscoveryService, ModuleRef } from '@nestjs/core';
 
@@ -17,6 +19,12 @@ interface KafkaModuleOptions {
     isGlobal?: boolean;
     /** Auto-create topics via admin on first use (send/consume). Useful for development. */
     autoCreateTopics?: boolean;
+    /** When `true`, string topic keys are validated against any schema previously registered via a TopicDescriptor. Default: `true`. */
+    strictSchemas?: boolean;
+    /** Number of partitions for auto-created topics. Default: `1`. */
+    numPartitions?: number;
+    /** Client-wide instrumentation hooks (e.g. OTel). Applied to both send and consume paths. */
+    instrumentation?: KafkaInstrumentation[];
 }
 /** Async configuration for `KafkaModule.registerAsync()` with dependency injection. */
 interface KafkaModuleAsyncOptions {
@@ -86,4 +94,4 @@ declare class KafkaHealthIndicator {
     check<T extends TopicMapConstraint<T>>(client: KafkaClient<T>): Promise<KafkaHealthResult>;
 }
 
-export { ClientId, ConsumerOptions, GroupId, InjectKafkaClient, KAFKA_CLIENT, KAFKA_SUBSCRIBER_METADATA, KafkaClient, KafkaExplorer, KafkaHealthIndicator, type KafkaHealthResult, KafkaModule, type KafkaModuleAsyncOptions, type KafkaModuleOptions, type KafkaSubscriberMetadata, SchemaLike, SubscribeTo, TopicDescriptor, TopicMapConstraint, getKafkaClientToken };
+export { ClientId, ConsumerOptions, GroupId, InjectKafkaClient, KAFKA_CLIENT, KAFKA_SUBSCRIBER_METADATA, KafkaClient, KafkaExplorer, KafkaHealthIndicator, type KafkaHealthResult, KafkaInstrumentation, KafkaModule, type KafkaModuleAsyncOptions, type KafkaModuleOptions, type KafkaSubscriberMetadata, SchemaLike, SubscribeTo, TopicDescriptor, TopicMapConstraint, getKafkaClientToken };