@drarzter/kafka-client 0.5.0 → 0.5.2

package/dist/{envelope-QK1trQu4.d.ts → envelope-C66_h8r_.d.ts}

@@ -9,7 +9,7 @@
  * ```
  */
 interface SchemaLike<T = any> {
-    parse(data: unknown): T;
+    parse(data: unknown): T | Promise<T>;
 }
 /** Infer the output type from a SchemaLike. */
 type InferSchema<S extends SchemaLike> = S extends SchemaLike<infer T> ? T : never;
@@ -158,8 +158,10 @@ interface ConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> {
 interface RetryOptions {
     /** Maximum number of retry attempts before giving up. */
     maxRetries: number;
-    /** Base delay between retries in ms (multiplied by attempt number). Default: `1000`. */
+    /** Base delay for exponential backoff in ms. Default: `1000`. */
     backoffMs?: number;
+    /** Maximum delay cap for exponential backoff in ms. Default: `30000`. */
+    maxBackoffMs?: number;
 }
 /**
  * Interceptor hooks for consumer message processing.
@@ -202,6 +204,8 @@ interface TransactionContext<T extends TopicMapConstraint<T>> {
 /** Interface describing all public methods of the Kafka client. */
 interface IKafkaClient<T extends TopicMapConstraint<T>> {
     checkStatus(): Promise<{
+        status: 'up';
+        clientId: string;
         topics: string[];
     }>;
     startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (envelope: EventEnvelope<T[K[number]]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
@@ -224,6 +228,20 @@ interface KafkaLogger {
     warn(message: string, ...args: any[]): void;
     error(message: string, ...args: any[]): void;
 }
+/**
+ * Context passed to `onMessageLost` when a message is silently dropped
+ * (handler threw and `dlq` is not enabled).
+ */
+interface MessageLostContext {
+    /** Topic the message was consumed from. */
+    topic: string;
+    /** Error that caused the message to be dropped. */
+    error: Error;
+    /** Number of processing attempts (0 = validation failure, before handler ran). */
+    attempt: number;
+    /** Original Kafka message headers (correlationId, traceparent, etc.). */
+    headers: MessageHeaders;
+}
 /** Options for `KafkaClient` constructor. */
 interface KafkaClientOptions {
     /** Auto-create topics via admin before the first `sendMessage`, `sendBatch`, or `transaction` for each topic. Useful for development — not recommended in production. */
@@ -236,6 +254,12 @@ interface KafkaClientOptions {
     numPartitions?: number;
     /** Client-wide instrumentation hooks (e.g. OTel). Applied to both send and consume paths. */
     instrumentation?: KafkaInstrumentation[];
+    /**
+     * Called when a message is dropped without being sent to a DLQ.
+     * Fires when the handler throws after all retries, or schema validation fails — and `dlq` is not enabled.
+     * Use this to alert, log to external systems, or trigger fallback logic.
+     */
+    onMessageLost?: (ctx: MessageLostContext) => void | Promise<void>;
 }
 /** Options for consumer subscribe retry when topic doesn't exist yet. */
 interface SubscribeRetryOptions {
@@ -316,4 +340,4 @@ declare function decodeHeaders(raw: Record<string, Buffer | string | (Buffer | s
  */
 declare function extractEnvelope<T>(payload: T, headers: MessageHeaders, topic: string, partition: number, offset: string): EventEnvelope<T>;
 
-export { type BatchMessageItem as B, type ClientId as C, type EnvelopeHeaderOptions as E, type GroupId as G, HEADER_CORRELATION_ID as H, type IKafkaClient as I, type KafkaInstrumentation as K, type MessageHeaders as M, type RetryOptions as R, type SchemaLike as S, type TopicMapConstraint as T, type ConsumerOptions as a, type TopicDescriptor as b, type BatchMeta as c, type ConsumerInterceptor as d, type EventEnvelope as e, HEADER_EVENT_ID as f, HEADER_SCHEMA_VERSION as g, HEADER_TIMESTAMP as h, HEADER_TRACEPARENT as i, type InferSchema as j, type KafkaClientOptions as k, type KafkaLogger as l, type SendOptions as m, type SubscribeRetryOptions as n, type TTopicMessageMap as o, type TopicsFrom as p, type TransactionContext as q, buildEnvelopeHeaders as r, decodeHeaders as s, extractEnvelope as t, getEnvelopeContext as u, runWithEnvelopeContext as v, topic as w };
+export { type BatchMessageItem as B, type ClientId as C, type EnvelopeHeaderOptions as E, type GroupId as G, HEADER_CORRELATION_ID as H, type IKafkaClient as I, type KafkaInstrumentation as K, type MessageHeaders as M, type RetryOptions as R, type SchemaLike as S, type TopicMapConstraint as T, type ConsumerOptions as a, type TopicDescriptor as b, type BatchMeta as c, type ConsumerInterceptor as d, type EventEnvelope as e, HEADER_EVENT_ID as f, HEADER_SCHEMA_VERSION as g, HEADER_TIMESTAMP as h, HEADER_TRACEPARENT as i, type InferSchema as j, type KafkaClientOptions as k, type KafkaLogger as l, type MessageLostContext as m, type SendOptions as n, type SubscribeRetryOptions as o, type TTopicMessageMap as p, type TopicsFrom as q, type TransactionContext as r, buildEnvelopeHeaders as s, decodeHeaders as t, extractEnvelope as u, getEnvelopeContext as v, runWithEnvelopeContext as w, topic as x };

package/dist/index.d.mts

@@ -1,7 +1,7 @@
 import { KafkaClient } from './core.mjs';
 export { KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError } from './core.mjs';
-import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, S as SchemaLike, a as ConsumerOptions, b as TopicDescriptor } from './envelope-QK1trQu4.mjs';
-export { B as BatchMessageItem, c as BatchMeta, d as ConsumerInterceptor, E as EnvelopeHeaderOptions, e as EventEnvelope, H as HEADER_CORRELATION_ID, f as HEADER_EVENT_ID, g as HEADER_SCHEMA_VERSION, h as HEADER_TIMESTAMP, i as HEADER_TRACEPARENT, I as IKafkaClient, j as InferSchema, k as KafkaClientOptions, l as KafkaLogger, M as MessageHeaders, R as RetryOptions, m as SendOptions, n as SubscribeRetryOptions, o as TTopicMessageMap, p as TopicsFrom, q as TransactionContext, r as buildEnvelopeHeaders, s as decodeHeaders, t as extractEnvelope, u as getEnvelopeContext, v as runWithEnvelopeContext, w as topic } from './envelope-QK1trQu4.mjs';
+import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, S as SchemaLike, a as ConsumerOptions, b as TopicDescriptor } from './envelope-C66_h8r_.mjs';
+export { B as BatchMessageItem, c as BatchMeta, d as ConsumerInterceptor, E as EnvelopeHeaderOptions, e as EventEnvelope, H as HEADER_CORRELATION_ID, f as HEADER_EVENT_ID, g as HEADER_SCHEMA_VERSION, h as HEADER_TIMESTAMP, i as HEADER_TRACEPARENT, I as IKafkaClient, j as InferSchema, k as KafkaClientOptions, l as KafkaLogger, M as MessageHeaders, m as MessageLostContext, R as RetryOptions, n as SendOptions, o as SubscribeRetryOptions, p as TTopicMessageMap, q as TopicsFrom, r as TransactionContext, s as buildEnvelopeHeaders, t as decodeHeaders, u as extractEnvelope, v as getEnvelopeContext, w as runWithEnvelopeContext, x as topic } from './envelope-C66_h8r_.mjs';
 import { DynamicModule, OnModuleInit } from '@nestjs/common';
 import { DiscoveryService, ModuleRef } from '@nestjs/core';
 

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 import { KafkaClient } from './core.js';
 export { KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError } from './core.js';
-import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, S as SchemaLike, a as ConsumerOptions, b as TopicDescriptor } from './envelope-QK1trQu4.js';
-export { B as BatchMessageItem, c as BatchMeta, d as ConsumerInterceptor, E as EnvelopeHeaderOptions, e as EventEnvelope, H as HEADER_CORRELATION_ID, f as HEADER_EVENT_ID, g as HEADER_SCHEMA_VERSION, h as HEADER_TIMESTAMP, i as HEADER_TRACEPARENT, I as IKafkaClient, j as InferSchema, k as KafkaClientOptions, l as KafkaLogger, M as MessageHeaders, R as RetryOptions, m as SendOptions, n as SubscribeRetryOptions, o as TTopicMessageMap, p as TopicsFrom, q as TransactionContext, r as buildEnvelopeHeaders, s as decodeHeaders, t as extractEnvelope, u as getEnvelopeContext, v as runWithEnvelopeContext, w as topic } from './envelope-QK1trQu4.js';
+import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, S as SchemaLike, a as ConsumerOptions, b as TopicDescriptor } from './envelope-C66_h8r_.js';
+export { B as BatchMessageItem, c as BatchMeta, d as ConsumerInterceptor, E as EnvelopeHeaderOptions, e as EventEnvelope, H as HEADER_CORRELATION_ID, f as HEADER_EVENT_ID, g as HEADER_SCHEMA_VERSION, h as HEADER_TIMESTAMP, i as HEADER_TRACEPARENT, I as IKafkaClient, j as InferSchema, k as KafkaClientOptions, l as KafkaLogger, M as MessageHeaders, m as MessageLostContext, R as RetryOptions, n as SendOptions, o as SubscribeRetryOptions, p as TTopicMessageMap, q as TopicsFrom, r as TransactionContext, s as buildEnvelopeHeaders, t as decodeHeaders, u as extractEnvelope, v as getEnvelopeContext, w as runWithEnvelopeContext, x as topic } from './envelope-C66_h8r_.js';
 import { DynamicModule, OnModuleInit } from '@nestjs/common';
 import { DiscoveryService, ModuleRef } from '@nestjs/core';
 

package/dist/index.js

@@ -172,7 +172,7 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
   const schema = schemaMap.get(topic2);
   if (!schema) return message;
   try {
-    return schema.parse(message);
+    return await schema.parse(message);
   } catch (error) {
     const err = toError(error);
     const validationError = new KafkaValidationError(topic2, message, {
@@ -182,20 +182,36 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
       `Schema validation failed for topic ${topic2}:`,
       err.message
     );
-    if (dlq) await sendToDlq(topic2, raw, deps);
-    const errorEnvelope = extractEnvelope(message, {}, topic2, -1, "");
+    if (dlq) {
+      await sendToDlq(topic2, raw, deps, {
+        error: validationError,
+        attempt: 0,
+        originalHeaders: deps.originalHeaders
+      });
+    } else {
+      await deps.onMessageLost?.({ topic: topic2, error: validationError, attempt: 0, headers: deps.originalHeaders ?? {} });
+    }
+    const errorEnvelope = extractEnvelope(message, deps.originalHeaders ?? {}, topic2, -1, "");
     for (const interceptor of interceptors) {
       await interceptor.onError?.(errorEnvelope, validationError);
     }
     return null;
   }
 }
-async function sendToDlq(topic2, rawMessage, deps) {
+async function sendToDlq(topic2, rawMessage, deps, meta) {
   const dlqTopic = `${topic2}.dlq`;
+  const headers = {
+    ...meta?.originalHeaders ?? {},
+    "x-dlq-original-topic": topic2,
+    "x-dlq-failed-at": (/* @__PURE__ */ new Date()).toISOString(),
+    "x-dlq-error-message": meta?.error.message ?? "unknown",
+    "x-dlq-error-stack": meta?.error.stack?.slice(0, 2e3) ?? "",
+    "x-dlq-attempt-count": String(meta?.attempt ?? 0)
+  };
   try {
     await deps.producer.send({
       topic: dlqTopic,
-      messages: [{ value: rawMessage }]
+      messages: [{ value: rawMessage, headers }]
     });
     deps.logger.warn(`Message sent to DLQ: ${dlqTopic}`);
   } catch (error) {
@@ -209,6 +225,7 @@ async function executeWithRetry(fn, ctx, deps) {
   const { envelope, rawMessages, interceptors, dlq, retry, isBatch } = ctx;
   const maxAttempts = retry ? retry.maxRetries + 1 : 1;
   const backoffMs = retry?.backoffMs ?? 1e3;
+  const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
   const envelopes = Array.isArray(envelope) ? envelope : [envelope];
   const topic2 = envelopes[0]?.topic ?? "unknown";
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
@@ -267,12 +284,25 @@ async function executeWithRetry(fn, ctx, deps) {
       );
       if (isLastAttempt) {
         if (dlq) {
+          const dlqMeta = {
+            error: err,
+            attempt,
+            originalHeaders: envelopes[0]?.headers
+          };
           for (const raw of rawMessages) {
-            await sendToDlq(topic2, raw, deps);
+            await sendToDlq(topic2, raw, deps, dlqMeta);
           }
+        } else {
+          await deps.onMessageLost?.({
+            topic: topic2,
+            error: err,
+            attempt,
+            headers: envelopes[0]?.headers ?? {}
+          });
         }
       } else {
-        await sleep(backoffMs * attempt);
+        const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
+        await sleep(Math.random() * cap);
       }
     }
   }
@@ -314,6 +344,7 @@ var KafkaClient = class {
   schemaRegistry = /* @__PURE__ */ new Map();
   runningConsumers = /* @__PURE__ */ new Map();
   instrumentation;
+  onMessageLost;
   isAdminConnected = false;
   clientId;
   constructor(clientId, groupId, brokers, options) {
@@ -328,6 +359,7 @@ var KafkaClient = class {
     this.strictSchemasEnabled = options?.strictSchemas ?? true;
     this.numPartitions = options?.numPartitions ?? 1;
     this.instrumentation = options?.instrumentation ?? [];
+    this.onMessageLost = options?.onMessageLost;
     this.kafka = new KafkaClass({
       kafkaJS: {
         clientId: this.clientId,
@@ -343,7 +375,7 @@ var KafkaClient = class {
     this.admin = this.kafka.admin();
   }
   async sendMessage(topicOrDesc, message, options = {}) {
-    const payload = this.buildSendPayload(topicOrDesc, [
+    const payload = await this.buildSendPayload(topicOrDesc, [
       {
         value: message,
         key: options.key,
@@ -360,7 +392,7 @@ var KafkaClient = class {
     }
   }
   async sendBatch(topicOrDesc, messages) {
-    const payload = this.buildSendPayload(topicOrDesc, messages);
+    const payload = await this.buildSendPayload(topicOrDesc, messages);
     await this.ensureTopic(payload.topic);
     await this.producer.send(payload);
     for (const inst of this.instrumentation) {
@@ -384,7 +416,7 @@ var KafkaClient = class {
     try {
       const ctx = {
         send: async (topicOrDesc, message, options = {}) => {
-          const payload = this.buildSendPayload(topicOrDesc, [
+          const payload = await this.buildSendPayload(topicOrDesc, [
             {
               value: message,
               key: options.key,
@@ -398,7 +430,7 @@ var KafkaClient = class {
           await tx.send(payload);
         },
         sendBatch: async (topicOrDesc, messages) => {
-          const payload = this.buildSendPayload(topicOrDesc, messages);
+          const payload = await this.buildSendPayload(topicOrDesc, messages);
           await this.ensureTopic(payload.topic);
           await tx.send(payload);
         }
@@ -429,7 +461,7 @@ var KafkaClient = class {
   }
   async startConsumer(topics, handleMessage, options = {}) {
     const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
-    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation };
+    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
     await consumer.run({
       eachMessage: async ({ topic: topic2, partition, message }) => {
         if (!message.value) {
@@ -439,6 +471,7 @@ var KafkaClient = class {
         const raw = message.value.toString();
         const parsed = parseJsonMessage(raw, topic2, this.logger);
         if (parsed === null) return;
+        const headers = decodeHeaders(message.headers);
         const validated = await validateWithSchema(
           parsed,
           raw,
@@ -446,10 +479,9 @@ var KafkaClient = class {
           schemaMap,
           interceptors,
           dlq,
-          deps
+          { ...deps, originalHeaders: headers }
         );
         if (validated === null) return;
-        const headers = decodeHeaders(message.headers);
         const envelope = extractEnvelope(
           validated,
           headers,
@@ -471,7 +503,7 @@ var KafkaClient = class {
   }
   async startBatchConsumer(topics, handleBatch, options = {}) {
     const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
-    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation };
+    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
     await consumer.run({
       eachBatch: async ({
         batch,
@@ -491,6 +523,7 @@ var KafkaClient = class {
           const raw = message.value.toString();
           const parsed = parseJsonMessage(raw, batch.topic, this.logger);
           if (parsed === null) continue;
+          const headers = decodeHeaders(message.headers);
           const validated = await validateWithSchema(
             parsed,
             raw,
@@ -498,10 +531,9 @@ var KafkaClient = class {
             schemaMap,
             interceptors,
             dlq,
-            deps
+            { ...deps, originalHeaders: headers }
           );
           if (validated === null) continue;
-          const headers = decodeHeaders(message.headers);
           envelopes.push(
             extractEnvelope(validated, headers, batch.topic, batch.partition, message.offset)
           );
@@ -542,14 +574,14 @@ var KafkaClient = class {
     this.runningConsumers.clear();
     this.logger.log("All consumers disconnected");
   }
-  /** Check broker connectivity and return available topics. */
+  /** Check broker connectivity and return status, clientId, and available topics. */
   async checkStatus() {
     if (!this.isAdminConnected) {
       await this.admin.connect();
       this.isAdminConnected = true;
     }
     const topics = await this.admin.listTopics();
-    return { topics };
+    return { status: "up", clientId: this.clientId, topics };
   }
   getClientId() {
     return this.clientId;
@@ -611,13 +643,13 @@ var KafkaClient = class {
     }
   }
   /** Validate message against schema. Pure — no side-effects on registry. */
-  validateMessage(topicOrDesc, message) {
+  async validateMessage(topicOrDesc, message) {
     if (topicOrDesc?.__schema) {
-      return topicOrDesc.__schema.parse(message);
+      return await topicOrDesc.__schema.parse(message);
     }
     if (this.strictSchemasEnabled && typeof topicOrDesc === "string") {
       const schema = this.schemaRegistry.get(topicOrDesc);
-      if (schema) return schema.parse(message);
+      if (schema) return await schema.parse(message);
     }
     return message;
   }
@@ -626,12 +658,11 @@ var KafkaClient = class {
    * Handles: topic resolution, schema registration, validation, JSON serialization,
    * envelope header generation, and instrumentation hooks.
    */
-  buildSendPayload(topicOrDesc, messages) {
+  async buildSendPayload(topicOrDesc, messages) {
     this.registerSchema(topicOrDesc);
     const topic2 = this.resolveTopicName(topicOrDesc);
-    return {
-      topic: topic2,
-      messages: messages.map((m) => {
+    const builtMessages = await Promise.all(
+      messages.map(async (m) => {
         const envelopeHeaders = buildEnvelopeHeaders({
           correlationId: m.correlationId,
           schemaVersion: m.schemaVersion,
@@ -642,12 +673,13 @@ var KafkaClient = class {
           inst.beforeSend?.(topic2, envelopeHeaders);
         }
         return {
-          value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
+          value: JSON.stringify(await this.validateMessage(topicOrDesc, m.value)),
           key: m.key ?? null,
           headers: envelopeHeaders
         };
       })
-    };
+    );
+    return { topic: topic2, messages: builtMessages };
   }
   /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
   async setupConsumer(topics, mode, options) {
@@ -917,12 +949,7 @@ var import_common4 = require("@nestjs/common");
 var KafkaHealthIndicator = class {
   async check(client) {
     try {
-      const { topics } = await client.checkStatus();
-      return {
-        status: "up",
-        clientId: client.clientId,
-        topics
-      };
+      return await client.checkStatus();
     } catch (error) {
       return {
         status: "down",