@taphealth/kafka 1.6.52 → 2.0.0

package/README.md

@@ -46,6 +46,22 @@ await producer.send({
 });
 ```
 
+`send()` now returns `boolean`:
+- `true` when publish succeeds
+- `false` when publish fails (defensive, non-throwing)
+
+Producer instances are long-lived and reused across calls. On process shutdown, call:
+
+```typescript
+await producer.disconnect();
+```
+
+Producer instrumentation:
+- Starts span: `kafka.produce <topic>`
+- Adds attributes: `messaging.system`, `messaging.destination.name`, `messaging.operation=publish`
+- Injects W3C trace headers into message headers
+- Records metric `kafka.produce.error` on publish failures
+
 ### Consuming Messages
 
 Extend KafkaConsumer to implement message handling logic:
@@ -66,3 +82,45 @@ export class YourEventConsumer extends KafkaConsumer<YourEventInterface> {
 const consumer = new YourEventConsumer(kafkaClient.client);
 await consumer.consume();
 ```
+
+Consumer error behavior:
+- Malformed/poison payloads are skipped (consumer moves forward)
+- `onMessage` errors are rethrown (Kafka retries; handlers must be idempotent)
+
+Advanced consume options:
+
+```typescript
+await consumer.consume({
+  autoCommit: false,
+  retry: {
+    maxRetries: 3,
+    backoffMs: (attempt) => attempt * 250,
+  },
+  onDeadLetter: async (data, message, err) => {
+    // optional custom dead-letter sink
+  },
+});
+```
+
+Handler context:
+- `onMessage` receives optional third arg `{ heartbeat, attempt }`
+- Use `heartbeat()` for long-running handlers
+- `attempt` starts at `1` and increments per retry
+
+Consumer instrumentation:
+- Starts span: `kafka.consume <topic>`
+- Extracts W3C trace headers from message headers
+- Adds attributes: `messaging.system`, `messaging.destination.name`, `messaging.operation=receive`, partition, offset
+- Records `kafka.consume.poison` for parse failures and `kafka.consume.handler_error` for handler failures
+
+Reserved env flag:
+
+```bash
+KAFKA_CONSUMER_DLQ_ENABLED=false
+```
+
+## Migration notes
+
+- `onMessage` can now be async safely; rejections are awaited and rethrown to KafkaJS.
+- To opt into crash-safe manual commits, pass `autoCommit: false` to `consume()`.
+- If you rely on retries beyond KafkaJS defaults, pass `retry` + `onDeadLetter` in `consume()` options.

package/dist/__tests__/consumer-behavior.test.js

@@ -0,0 +1,284 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const consumer_1 = require("../consumer");
+const topics_1 = require("../topics");
+const assert = (condition, message) => {
+    if (!condition) {
+        throw new Error(message);
+    }
+};
+class ConsumerStub {
+    constructor() {
+        this.connectCalls = 0;
+        this.subscribeCalls = 0;
+        this.disconnectCalls = 0;
+        this.commitCalls = [];
+        this.commitShouldThrow = false;
+    }
+    async connect() {
+        this.connectCalls += 1;
+    }
+    async subscribe() {
+        this.subscribeCalls += 1;
+    }
+    async run(config) {
+        this.eachMessage = config.eachMessage;
+        this.runOptions = {
+            autoCommit: config.autoCommit,
+        };
+    }
+    async commitOffsets(offsets) {
+        if (this.commitShouldThrow) {
+            throw new Error("commit failed (rebalance)");
+        }
+        this.commitCalls.push(...offsets);
+    }
+    async disconnect() {
+        this.disconnectCalls += 1;
+    }
+}
+class KafkaStub {
+    constructor() {
+        this.consumerInstance = new ConsumerStub();
+    }
+    consumer() {
+        return this.consumerInstance;
+    }
+}
+class TestConsumer extends consumer_1.KafkaConsumer {
+    constructor() {
+        super(...arguments);
+        this.topic = topics_1.Topics.SubscriptionCreated;
+        this.groupId = "test-group";
+        this.shouldThrow = false;
+        this.shouldRejectAsync = false;
+        this.remainingFailures = 0;
+        this.heartbeatCalls = 0;
+        this.attempts = [];
+        this.received = [];
+    }
+    async onMessage(data, _msg, context) {
+        if (context) {
+            this.attempts.push(context.attempt);
+            await context.heartbeat();
+            this.heartbeatCalls += 1;
+        }
+        if (this.remainingFailures > 0) {
+            this.remainingFailures -= 1;
+            throw new Error("handler temporary failure");
+        }
+        if (this.shouldThrow) {
+            throw new Error("handler failed");
+        }
+        if (this.shouldRejectAsync) {
+            return Promise.reject(new Error("handler failed async"));
+        }
+        this.received.push(data);
+    }
+}
+const run = async () => {
+    var _a;
+    const kafka = new KafkaStub();
+    const consumer = new TestConsumer(kafka);
+    await consumer.consume({ autoCommit: false });
+    assert(kafka.consumerInstance.connectCalls === 1, "consumer should connect once");
+    assert(kafka.consumerInstance.subscribeCalls === 1, "consumer should subscribe once");
+    assert(!!kafka.consumerInstance.eachMessage, "eachMessage handler should be registered");
+    assert(((_a = kafka.consumerInstance.runOptions) === null || _a === void 0 ? void 0 : _a.autoCommit) === false, "consume options should forward autoCommit value to kafkajs");
+    await kafka.consumerInstance.eachMessage({
+        message: {
+            value: { toString: () => '{"id":"ok"}' },
+            offset: "1",
+        },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(consumer.received.length === 1, "valid message should reach handler");
+    assert(kafka.consumerInstance.commitCalls.length === 1 &&
+        kafka.consumerInstance.commitCalls[0].offset === "2" &&
+        kafka.consumerInstance.commitCalls[0].partition === 0, "autoCommit:false success path should manually commit next offset (message.offset + 1)");
+    await kafka.consumerInstance.eachMessage({
+        message: {
+            value: { toString: () => "not-json" },
+            offset: "2",
+        },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(consumer.received.length === 1, "poison message should be skipped");
+    assert(kafka.consumerInstance.commitCalls.length === 2 &&
+        kafka.consumerInstance.commitCalls[1].offset === "3", "autoCommit:false poison path should manually commit to advance past skipped message");
+    consumer.shouldThrow = true;
+    const commitsBeforeThrow = kafka.consumerInstance.commitCalls.length;
+    let handlerThrew = false;
+    try {
+        await kafka.consumerInstance.eachMessage({
+            message: {
+                value: { toString: () => '{"id":"boom"}' },
+                offset: "3",
+            },
+            partition: 0,
+            heartbeat: async () => undefined,
+        });
+    }
+    catch (_err) {
+        handlerThrew = true;
+    }
+    assert(handlerThrew, "handler errors should be rethrown");
+    assert(kafka.consumerInstance.commitCalls.length === commitsBeforeThrow, "autoCommit:false handler-error path with no DLQ should NOT commit (kafkajs will redeliver)");
+    consumer.shouldThrow = false;
+    consumer.shouldRejectAsync = true;
+    let asyncHandlerThrew = false;
+    try {
+        await kafka.consumerInstance.eachMessage({
+            message: {
+                value: { toString: () => '{"id":"boom-async"}' },
+                offset: "4",
+            },
+            partition: 0,
+            heartbeat: async () => undefined,
+        });
+    }
+    catch (_err) {
+        asyncHandlerThrew = true;
+    }
+    assert(asyncHandlerThrew, "async handler rejections should be rethrown");
+    assert(kafka.consumerInstance.commitCalls.length === commitsBeforeThrow, "autoCommit:false async-rejection path with no DLQ should NOT commit");
+    consumer.shouldRejectAsync = false;
+    consumer.remainingFailures = 2;
+    const deadLetterCalls = [];
+    await consumer.consume({
+        retry: {
+            maxRetries: 3,
+            backoffMs: () => 0,
+        },
+        onDeadLetter: async (data) => {
+            deadLetterCalls.push(data);
+        },
+    });
+    await kafka.consumerInstance.eachMessage({
+        message: {
+            value: { toString: () => '{"id":"retry-ok"}' },
+            offset: "5",
+        },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(consumer.received.some((item) => item.id === "retry-ok"), "consumer should succeed after configured retries");
+    assert(consumer.attempts.includes(1) && consumer.attempts.includes(3), "consumer should pass attempt count to handler context");
+    assert(consumer.heartbeatCalls > 0, "consumer should pass heartbeat to handler context");
+    consumer.remainingFailures = 2;
+    await consumer.consume({
+        retry: {
+            maxRetries: 1,
+            backoffMs: () => 0,
+        },
+        onDeadLetter: async (data) => {
+            deadLetterCalls.push(data);
+        },
+    });
+    await kafka.consumerInstance.eachMessage({
+        message: {
+            value: { toString: () => '{"id":"dead-letter"}' },
+            offset: "6",
+        },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(deadLetterCalls.some((item) => item.id === "dead-letter"), "onDeadLetter should be called after retry exhaustion");
+    consumer.remainingFailures = 2;
+    let deadLetterHookThrowSurfaces = false;
+    try {
+        await consumer.consume({
+            retry: {
+                maxRetries: 1,
+                backoffMs: () => 0,
+            },
+            onDeadLetter: async () => {
+                throw new Error("dead-letter hook failed");
+            },
+        });
+        await kafka.consumerInstance.eachMessage({
+            message: {
+                value: { toString: () => '{"id":"dead-letter-hook-throws"}' },
+                offset: "7",
+            },
+            partition: 0,
+            heartbeat: async () => undefined,
+        });
+    }
+    catch (_err) {
+        deadLetterHookThrowSurfaces = true;
+    }
+    assert(deadLetterHookThrowSurfaces, "consumer should rethrow when dead-letter hook itself throws");
+    // ─── autoCommit:false interaction with retry + DLQ + commit failures ──
+    const manualCommitConsumer = new TestConsumer(kafka);
+    // Reset commit log so subsequent assertions are independent.
+    kafka.consumerInstance.commitCalls = [];
+    manualCommitConsumer.remainingFailures = 2;
+    await manualCommitConsumer.consume({
+        autoCommit: false,
+        retry: { maxRetries: 3, backoffMs: () => 0 },
+        onDeadLetter: async () => undefined,
+    });
+    await kafka.consumerInstance.eachMessage({
+        message: { value: { toString: () => '{"id":"retry-then-commit"}' }, offset: "10" },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(kafka.consumerInstance.commitCalls.length === 1 &&
+        kafka.consumerInstance.commitCalls[0].offset === "11", "autoCommit:false should commit ONCE after retry-then-success (one commit per message, not per attempt)");
+    // autoCommit:false + retry-exhaustion + DLQ-handled → must commit so the
+    // failed message does not get redelivered forever.
+    kafka.consumerInstance.commitCalls = [];
+    manualCommitConsumer.remainingFailures = 5;
+    let dlqHandled = false;
+    await manualCommitConsumer.consume({
+        autoCommit: false,
+        retry: { maxRetries: 1, backoffMs: () => 0 },
+        onDeadLetter: async () => {
+            dlqHandled = true;
+        },
+    });
+    await kafka.consumerInstance.eachMessage({
+        message: { value: { toString: () => '{"id":"dlq-then-commit"}' }, offset: "20" },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(dlqHandled, "DLQ hook should fire on retry exhaustion");
+    assert(kafka.consumerInstance.commitCalls.length === 1 &&
+        kafka.consumerInstance.commitCalls[0].offset === "21", "autoCommit:false should commit after a DLQ-handled message so it is not redelivered");
+    // commitOffsets failure (rebalance, broker down) MUST be swallowed on the
+    // happy path — at-least-once is preserved by kafkajs replay on the next
+    // session, but propagating would force redelivery + perma-failure loop.
+    kafka.consumerInstance.commitCalls = [];
+    kafka.consumerInstance.commitShouldThrow = true;
+    manualCommitConsumer.remainingFailures = 0; // ensure handler succeeds cleanly
+    let commitFailureSurfaced = false;
+    try {
+        await kafka.consumerInstance.eachMessage({
+            message: { value: { toString: () => '{"id":"commit-fails-but-handler-ok"}' }, offset: "30" },
+            partition: 0,
+            heartbeat: async () => undefined,
+        });
+    }
+    catch (_err) {
+        commitFailureSurfaced = true;
+    }
+    assert(!commitFailureSurfaced, "commitOffsets failure must NOT propagate out of eachMessage (would cause kafkajs to redeliver and never commit)");
+    kafka.consumerInstance.commitShouldThrow = false;
+    // autoCommit:true (or undefined) → kafkajs handles commits, package must NOT
+    // call commitOffsets manually (would double-commit).
+    kafka.consumerInstance.commitCalls = [];
+    const autoCommitConsumer = new TestConsumer(kafka);
+    await autoCommitConsumer.consume({ autoCommit: true });
+    await kafka.consumerInstance.eachMessage({
+        message: { value: { toString: () => '{"id":"auto-commit-no-manual"}' }, offset: "40" },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(kafka.consumerInstance.commitCalls.length === 0, "autoCommit:true must NOT trigger manual commitOffsets (kafkajs owns commit cadence)");
+    await consumer.disconnect();
+    assert(kafka.consumerInstance.disconnectCalls === 1, "disconnect should use the running consumer instance");
+};
+run();

package/dist/__tests__/producer-behavior.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/producer-behavior.test.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const api_1 = require("@opentelemetry/api");
+const producer_1 = require("../producer");
+const topics_1 = require("../topics");
+const assert = (condition, message) => {
+    if (!condition) {
+        throw new Error(message);
+    }
+};
+class ProducerStub {
+    constructor() {
+        this.connectCalls = 0;
+        this.disconnectCalls = 0;
+        this.sendCalls = 0;
+        this.shouldFail = false;
+        this.payloads = [];
+    }
+    async connect() {
+        this.connectCalls += 1;
+    }
+    async disconnect() {
+        this.disconnectCalls += 1;
+    }
+    async send(payload) {
+        this.sendCalls += 1;
+        this.payloads.push(payload);
+        if (this.shouldFail) {
+            throw new Error("send failed");
+        }
+    }
+}
+class KafkaStub {
+    constructor() {
+        this.producerFactoryCalls = 0;
+        this.producerInstance = new ProducerStub();
+    }
+    producer() {
+        this.producerFactoryCalls += 1;
+        return this.producerInstance;
+    }
+}
+class TestProducer extends producer_1.KafkaProducer {
+    constructor() {
+        super(...arguments);
+        this.topic = topics_1.Topics.SubscriptionCreated;
+    }
+}
+const run = async () => {
+    const kafka = new KafkaStub();
+    const loggerErrors = [];
+    const logger = {
+        info: () => undefined,
+        error: (_message, meta) => {
+            loggerErrors.push(meta);
+        },
+    };
+    const producer = new TestProducer(kafka, logger);
+    const activeContext = api_1.propagation.setBaggage(api_1.context.active(), api_1.propagation.createBaggage({
+        "user.id": { value: "abc" },
+    }));
+    const first = await api_1.context.with(activeContext, async () => {
+        return producer.send({ id: "1" });
+    });
+    const second = await producer.send({ id: "2" });
+    assert(first === true, "send should return true on first success");
+    assert(second === true, "send should return true on second success");
+    assert(kafka.producerFactoryCalls === 1, "producer should be created once and reused");
+    assert(kafka.producerInstance.connectCalls === 1, "producer connect should happen once");
+    assert(kafka.producerInstance.sendCalls === 2, "producer send should happen for each call");
+    const firstPayload = kafka.producerInstance.payloads[0];
+    assert(!!firstPayload.messages[0].headers, "producer should inject trace headers on outgoing messages");
+    kafka.producerInstance.shouldFail = true;
+    const failed = await producer.send({ id: "3" });
+    assert(failed === false, "send should return false on producer send error");
+    assert(loggerErrors.length === 1, "producer error should be logged once");
+    await producer.disconnect();
+    assert(kafka.producerInstance.disconnectCalls === 1, "disconnect should disconnect cached producer");
+};
+run();

package/dist/__tests__/weekly-command-contract.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/consumer.d.ts

@@ -1,16 +1,53 @@
 import { Kafka, Message } from "kafkajs";
+import { Logger } from "./logger";
 interface Event {
     topic: string;
     data: any;
 }
+export interface ConsumeRetryOptions {
+    maxRetries: number;
+    backoffMs: (attempt: number) => number;
+}
+export interface ConsumeHandlerContext {
+    heartbeat: () => Promise<void>;
+    attempt: number;
+}
+export interface ConsumeOptions<TData> {
+    autoCommit?: boolean;
+    retry?: ConsumeRetryOptions;
+    onDeadLetter?: (data: TData, message: Message, err: Error) => Promise<void> | void;
+}
 export declare abstract class KafkaConsumer<T extends Event> {
     abstract topic: T["topic"];
     abstract groupId: string;
-    abstract onMessage(data: T["data"], msg: Message): void;
+    abstract onMessage(data: T["data"], msg: Message, context?: ConsumeHandlerContext): void | Promise<void>;
     protected client: Kafka;
-    constructor(client: Kafka);
-    consume(): Promise<void>;
-    parseMessage(msg: Message): any;
+    private consumerInstance?;
+    private logger;
+    constructor(client: Kafka, logger?: Logger);
+    private getConsumer;
+    /**
+     * Error handling behavior:
+     * - Poison parse errors are skipped so offsets can move forward.
+     * - Handler errors are rethrown so kafkajs retries message processing.
+     */
+    private executeWithRetry;
+    /**
+     * Commit the offset for a single processed message. Used when callers opt
+     * into manual commits via `autoCommit: false`. Failures are logged and
+     * swallowed: at-least-once delivery is preserved because kafkajs will
+     * resume from the last successfully committed offset on the next session.
+     */
+    private commitMessageOffset;
+    /**
+     * Error handling behavior:
+     * - Poison parse errors are skipped so offsets can move forward.
+     * - Handler errors are rethrown so kafkajs retries message processing.
+     * - With `autoCommit: false`, offsets are manually committed after a
+     *   message is fully handled (success, poison-skip, or DLQ-handled).
+     */
+    consume(options?: ConsumeOptions<T["data"]>): Promise<void>;
+    parseMessage(msg: Message): T["data"];
     disconnect(): Promise<void>;
 }
 export {};

package/dist/consumer.js

@@ -1,21 +1,180 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.KafkaConsumer = void 0;
+const api_1 = require("@opentelemetry/api");
+const logger_1 = require("./logger");
+const telemetry_1 = require("./telemetry");
+const toError = (err) => {
+    if (err instanceof Error) {
+        return err;
+    }
+    return new Error(String(err));
+};
+const sleep = async (ms) => new Promise((resolve) => {
+    setTimeout(resolve, ms);
+});
 class KafkaConsumer {
-    constructor(client) {
+    constructor(client, logger = logger_1.defaultLogger) {
         this.client = client;
+        this.logger = logger;
+    }
+    getConsumer() {
+        if (!this.consumerInstance) {
+            this.consumerInstance = this.client.consumer({
+                groupId: this.groupId,
+            });
+        }
+        return this.consumerInstance;
     }
-    async consume() {
-        const consumer = this.client.consumer({
-            groupId: this.groupId,
+    /**
+     * Error handling behavior:
+     * - Poison parse errors are skipped so offsets can move forward.
+     * - Handler errors are rethrown so kafkajs retries message processing.
+     */
+    async executeWithRetry(data, message, heartbeat, options) {
+        var _a, _b, _c, _d;
+        const maxRetries = (_b = (_a = options === null || options === void 0 ? void 0 : options.retry) === null || _a === void 0 ? void 0 : _a.maxRetries) !== null && _b !== void 0 ? _b : 0;
+        const backoffMs = (_d = (_c = options === null || options === void 0 ? void 0 : options.retry) === null || _c === void 0 ? void 0 : _c.backoffMs) !== null && _d !== void 0 ? _d : (() => {
+            return 0;
         });
+        for (let attempt = 1; attempt <= maxRetries + 1; attempt += 1) {
+            try {
+                await this.onMessage(data, message, { heartbeat, attempt });
+                return;
+            }
+            catch (err) {
+                const error = toError(err);
+                (0, telemetry_1.incrementConsumeHandlerError)(this.topic);
+                if (attempt <= maxRetries) {
+                    await sleep(backoffMs(attempt));
+                    continue;
+                }
+                throw error;
+            }
+        }
+    }
+    /**
+     * Commit the offset for a single processed message. Used when callers opt
+     * into manual commits via `autoCommit: false`. Failures are logged and
+     * swallowed: at-least-once delivery is preserved because kafkajs will
+     * resume from the last successfully committed offset on the next session.
+     */
+    async commitMessageOffset(consumer, partition, offset) {
+        try {
+            const nextOffset = (BigInt(offset) + BigInt(1)).toString();
+            await consumer.commitOffsets([
+                {
+                    topic: this.topic,
+                    partition,
+                    offset: nextOffset,
+                },
+            ]);
+        }
+        catch (err) {
+            this.logger.error("Failed to commit kafka offset", {
+                topic: this.topic,
+                partition,
+                offset,
+                error: toError(err),
+            });
+        }
+    }
+    /**
+     * Error handling behavior:
+     * - Poison parse errors are skipped so offsets can move forward.
+     * - Handler errors are rethrown so kafkajs retries message processing.
+     * - With `autoCommit: false`, offsets are manually committed after a
+     *   message is fully handled (success, poison-skip, or DLQ-handled).
+     */
+    async consume(options) {
+        const consumer = this.getConsumer();
+        const manualCommit = (options === null || options === void 0 ? void 0 : options.autoCommit) === false;
         await consumer.connect();
         await consumer.subscribe({ topics: [this.topic], fromBeginning: true });
+        const dlqEnabled = process.env.KAFKA_CONSUMER_DLQ_ENABLED === "true";
+        if (dlqEnabled) {
+            this.logger.info("KAFKA_CONSUMER_DLQ_ENABLED is set (reserved, no-op)", {
+                topic: this.topic,
+            });
+        }
         await consumer.run({
-            eachMessage: async ({ message, partition }) => {
-                console.log("Received message on topic", this.topic, " and partition", partition);
-                const data = this.parseMessage(message);
-                this.onMessage(data, message);
+            autoCommit: options === null || options === void 0 ? void 0 : options.autoCommit,
+            eachMessage: async ({ message, partition, heartbeat }) => {
+                const extractionContext = (0, telemetry_1.extractContextFromMessage)(message);
+                await telemetry_1.kafkaTracer.startActiveSpan(`kafka.consume ${this.topic}`, {
+                    attributes: {
+                        "messaging.system": "kafka",
+                        "messaging.destination.name": this.topic,
+                        "messaging.operation": "receive",
+                        "messaging.kafka.partition": partition,
+                        "messaging.kafka.offset": message.offset,
+                    },
+                }, extractionContext, async (span) => {
+                    this.logger.info("Received message on topic", {
+                        topic: this.topic,
+                        partition,
+                        offset: message.offset,
+                    });
+                    let data;
+                    try {
+                        data = this.parseMessage(message);
+                    }
+                    catch (err) {
+                        const error = toError(err);
+                        (0, telemetry_1.incrementConsumePoison)(this.topic);
+                        span.recordException(error);
+                        span.setStatus({ code: api_1.SpanStatusCode.ERROR });
+                        this.logger.error("Skipping poison kafka message", {
+                            topic: this.topic,
+                            partition,
+                            offset: message.offset,
+                            error,
+                        });
+                        if (manualCommit) {
+                            await this.commitMessageOffset(consumer, partition, message.offset);
+                        }
+                        span.end();
+                        return;
+                    }
+                    try {
+                        await this.executeWithRetry(data, message, heartbeat, options);
+                    }
+                    catch (err) {
+                        const error = toError(err);
+                        span.recordException(error);
+                        span.setStatus({ code: api_1.SpanStatusCode.ERROR });
+                        this.logger.error("Kafka consumer handler failed", {
+                            topic: this.topic,
+                            partition,
+                            offset: message.offset,
+                            error,
+                        });
+                        if (options === null || options === void 0 ? void 0 : options.onDeadLetter) {
+                            try {
+                                await options.onDeadLetter(data, message, error);
+                                if (manualCommit) {
+                                    await this.commitMessageOffset(consumer, partition, message.offset);
+                                }
+                                span.end();
+                                return;
+                            }
+                            catch (deadLetterError) {
+                                this.logger.error("Kafka consumer dead-letter hook failed", {
+                                    topic: this.topic,
+                                    partition,
+                                    offset: message.offset,
+                                    error: toError(deadLetterError),
+                                });
+                            }
+                        }
+                        span.end();
+                        throw error;
+                    }
+                    if (manualCommit) {
+                        await this.commitMessageOffset(consumer, partition, message.offset);
+                    }
+                    span.end();
+                });
             },
         });
     }
@@ -25,7 +184,11 @@ class KafkaConsumer {
         return data;
     }
     async disconnect() {
-        await this.client.consumer({ groupId: this.groupId }).disconnect();
+        if (!this.consumerInstance) {
+            return;
+        }
+        await this.consumerInstance.disconnect();
+        this.consumerInstance = undefined;
     }
 }
 exports.KafkaConsumer = KafkaConsumer;

package/dist/index.d.ts

@@ -2,6 +2,7 @@ export * from "./topics";
 export * from "./kafka";
 export * from "./producer";
 export * from "./consumer";
+export * from "./logger";
 export * from "./types/appointment";
 export * from "./types/payment";
 export * from "./types/subscription";

package/dist/index.js

@@ -18,6 +18,7 @@ __exportStar(require("./topics"), exports);
 __exportStar(require("./kafka"), exports);
 __exportStar(require("./producer"), exports);
 __exportStar(require("./consumer"), exports);
+__exportStar(require("./logger"), exports);
 __exportStar(require("./types/appointment"), exports);
 __exportStar(require("./types/payment"), exports);
 __exportStar(require("./types/subscription"), exports);

package/dist/logger.d.ts

@@ -0,0 +1,5 @@
+export interface Logger {
+    info(message: string, meta?: Record<string, unknown>): void;
+    error(message: string, meta?: Record<string, unknown>): void;
+}
+export declare const defaultLogger: Logger;

package/dist/logger.js

@@ -0,0 +1,24 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultLogger = void 0;
+const middlewares_1 = require("@taphealth/middlewares");
+const winston_1 = __importDefault(require("winston"));
+const logger = winston_1.default.createLogger({
+    level: process.env.LOG_LEVEL || "info",
+    format: winston_1.default.format.combine(winston_1.default.format.timestamp(), winston_1.default.format.errors({ stack: true }), winston_1.default.format.json()),
+    transports: [
+        new winston_1.default.transports.Console(),
+        (0, middlewares_1.createOtelWinstonTransport)(),
+    ],
+});
+exports.defaultLogger = {
+    info(message, meta) {
+        logger.info(message, meta !== null && meta !== void 0 ? meta : {});
+    },
+    error(message, meta) {
+        logger.error(message, meta !== null && meta !== void 0 ? meta : {});
+    },
+};

package/dist/producer.d.ts

@@ -1,4 +1,5 @@
 import { Kafka } from "kafkajs";
+import { Logger } from "./logger";
 import { Topics } from "./topics";
 interface Event {
     topic: Topics;
@@ -7,7 +8,12 @@ interface Event {
 export declare abstract class KafkaProducer<T extends Event> {
     abstract topic: T["topic"];
     protected client: Kafka;
-    constructor(client: Kafka);
-    send(data: T["data"]): Promise<void>;
+    private producerInstance?;
+    private isProducerConnected;
+    private logger;
+    constructor(client: Kafka, logger?: Logger);
+    private getProducer;
+    send(data: T["data"]): Promise<boolean>;
+    disconnect(): Promise<void>;
 }
 export {};

package/dist/producer.js

@@ -1,27 +1,71 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.KafkaProducer = void 0;
+const api_1 = require("@opentelemetry/api");
+const logger_1 = require("./logger");
+const telemetry_1 = require("./telemetry");
 class KafkaProducer {
-    constructor(client) {
+    constructor(client, logger = logger_1.defaultLogger) {
+        this.isProducerConnected = false;
         this.client = client;
+        this.logger = logger;
     }
-    async send(data) {
-        try {
-            const producer = this.client.producer();
-            await producer.connect();
-            await producer.send({
-                topic: this.topic,
-                messages: [{
-                        partition: 0,
-                        value: JSON.stringify(data)
-                    }],
-            });
-            console.log("Data sent to Kafka topic", this.topic);
-            await producer.disconnect();
+    getProducer() {
+        if (!this.producerInstance) {
+            this.producerInstance = this.client.producer();
         }
-        catch (err) {
-            console.log("Error in sending data to Kafka", err);
+        return this.producerInstance;
+    }
+    async send(data) {
+        return telemetry_1.kafkaTracer.startActiveSpan(`kafka.produce ${this.topic}`, {
+            attributes: {
+                "messaging.system": "kafka",
+                "messaging.destination.name": this.topic,
+                "messaging.operation": "publish",
+            },
+        }, async (span) => {
+            try {
+                const producer = this.getProducer();
+                if (!this.isProducerConnected) {
+                    await producer.connect();
+                    this.isProducerConnected = true;
+                }
+                await producer.send({
+                    topic: this.topic,
+                    messages: [
+                        {
+                            partition: 0,
+                            value: JSON.stringify(data),
+                            headers: (0, telemetry_1.injectTraceHeaders)(),
+                        },
+                    ],
+                });
+                this.logger.info("Data sent to Kafka topic", { topic: this.topic });
+                return true;
+            }
+            catch (err) {
+                span.recordException(err);
+                span.setStatus({ code: api_1.SpanStatusCode.ERROR });
+                (0, telemetry_1.incrementProduceError)(this.topic);
+                this.logger.error("Error in sending data to Kafka", {
+                    topic: this.topic,
+                    error: err,
+                });
+                this.isProducerConnected = false;
+                return false;
+            }
+            finally {
+                span.end();
+            }
+        });
+    }
+    async disconnect() {
+        if (!this.producerInstance) {
+            return;
         }
+        await this.producerInstance.disconnect();
+        this.isProducerConnected = false;
+        this.producerInstance = undefined;
     }
 }
 exports.KafkaProducer = KafkaProducer;

package/dist/telemetry.d.ts

@@ -0,0 +1,9 @@
+import { Message } from "kafkajs";
+export declare const kafkaTracer: import("@opentelemetry/api").Tracer;
+export declare const injectTraceHeaders: (initialHeaders?: Record<string, string>) => {
+    [x: string]: string;
+};
+export declare const extractContextFromMessage: (message: Message) => import("@opentelemetry/api").Context;
+export declare const incrementProduceError: (topic: string) => void;
+export declare const incrementConsumePoison: (topic: string) => void;
+export declare const incrementConsumeHandlerError: (topic: string) => void;

package/dist/telemetry.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.incrementConsumeHandlerError = exports.incrementConsumePoison = exports.incrementProduceError = exports.extractContextFromMessage = exports.injectTraceHeaders = exports.kafkaTracer = void 0;
+const api_1 = require("@opentelemetry/api");
+exports.kafkaTracer = api_1.trace.getTracer("@taphealth/kafka");
+const kafkaMeter = api_1.metrics.getMeter("@taphealth/kafka");
+const produceErrorCounter = kafkaMeter.createCounter("kafka.produce.error", {
+    description: "Count of kafka produce failures",
+});
+const consumePoisonCounter = kafkaMeter.createCounter("kafka.consume.poison", {
+    description: "Count of kafka poison messages skipped during parsing",
+});
+const consumeHandlerErrorCounter = kafkaMeter.createCounter("kafka.consume.handler_error", {
+    description: "Count of kafka consumer handler failures",
+});
+const toHeaderValue = (value) => {
+    if (!value) {
+        return "";
+    }
+    if (Array.isArray(value)) {
+        const [first] = value;
+        return toHeaderValue(first);
+    }
+    if (typeof value === "string") {
+        return value;
+    }
+    return value.toString("utf8");
+};
+const injectTraceHeaders = (initialHeaders = {}) => {
+    const headers = Object.assign({}, initialHeaders);
+    api_1.propagation.inject(api_1.context.active(), headers);
+    return headers;
+};
+exports.injectTraceHeaders = injectTraceHeaders;
+const extractContextFromMessage = (message) => {
+    var _a;
+    const carrier = Object.entries((_a = message.headers) !== null && _a !== void 0 ? _a : {}).reduce((acc, [key, value]) => {
+        const headerValue = toHeaderValue(value);
+        if (headerValue) {
+            acc[key] = headerValue;
+        }
+        return acc;
+    }, {});
+    return api_1.propagation.extract(api_1.context.active(), carrier);
+};
+exports.extractContextFromMessage = extractContextFromMessage;
+const incrementProduceError = (topic) => {
+    produceErrorCounter.add(1, { topic });
+};
+exports.incrementProduceError = incrementProduceError;
+const incrementConsumePoison = (topic) => {
+    consumePoisonCounter.add(1, { topic });
+};
+exports.incrementConsumePoison = incrementConsumePoison;
+const incrementConsumeHandlerError = (topic) => {
+    consumeHandlerErrorCounter.add(1, { topic });
+};
+exports.incrementConsumeHandlerError = incrementConsumeHandlerError;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@taphealth/kafka",
-  "version": "1.6.52",
+  "version": "2.0.0",
   "description": "",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -16,13 +16,19 @@
     "typescript": "^5.4.5"
   },
   "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
     "@taphealth/errors": "^1.1.8",
-    "kafkajs": "^2.2.4"
+    "@taphealth/middlewares": "^1.2.13",
+    "kafkajs": "^2.2.4",
+    "winston": "^3.19.0"
   },
   "scripts": {
     "clean": "del-cli ./dist/*",
     "build": "pnpm clean && tsc",
-    "test:contracts": "ts-node ./src/tests/weekly-command-contract.test.ts",
+    "test:producer": "ts-node ./src/__tests__/producer-behavior.test.ts",
+    "test:consumer": "ts-node ./src/__tests__/consumer-behavior.test.ts",
+    "test": "pnpm test:contracts && pnpm test:producer && pnpm test:consumer",
+    "test:contracts": "ts-node ./src/__tests__/weekly-command-contract.test.ts",
     "pub": "pnpm version patch --no-git-checks && pnpm build && pnpm publish --no-git-checks"
   }
 }