@taphealth/kafka 1.6.53 → 2.0.0

package/dist/__tests__/consumer-behavior.test.js

@@ -12,6 +12,8 @@ class ConsumerStub {
         this.connectCalls = 0;
         this.subscribeCalls = 0;
         this.disconnectCalls = 0;
+        this.commitCalls = [];
+        this.commitShouldThrow = false;
     }
     async connect() {
         this.connectCalls += 1;
@@ -25,6 +27,12 @@ class ConsumerStub {
             autoCommit: config.autoCommit,
         };
     }
+    async commitOffsets(offsets) {
+        if (this.commitShouldThrow) {
+            throw new Error("commit failed (rebalance)");
+        }
+        this.commitCalls.push(...offsets);
+    }
     async disconnect() {
         this.disconnectCalls += 1;
     }
@@ -86,6 +94,9 @@ const run = async () => {
         heartbeat: async () => undefined,
     });
     assert(consumer.received.length === 1, "valid message should reach handler");
+    assert(kafka.consumerInstance.commitCalls.length === 1 &&
+        kafka.consumerInstance.commitCalls[0].offset === "2" &&
+        kafka.consumerInstance.commitCalls[0].partition === 0, "autoCommit:false success path should manually commit next offset (message.offset + 1)");
     await kafka.consumerInstance.eachMessage({
         message: {
             value: { toString: () => "not-json" },
@@ -95,7 +106,10 @@ const run = async () => {
         heartbeat: async () => undefined,
     });
     assert(consumer.received.length === 1, "poison message should be skipped");
+    assert(kafka.consumerInstance.commitCalls.length === 2 &&
+        kafka.consumerInstance.commitCalls[1].offset === "3", "autoCommit:false poison path should manually commit to advance past skipped message");
     consumer.shouldThrow = true;
+    const commitsBeforeThrow = kafka.consumerInstance.commitCalls.length;
     let handlerThrew = false;
     try {
         await kafka.consumerInstance.eachMessage({
@@ -111,6 +125,7 @@ const run = async () => {
         handlerThrew = true;
     }
     assert(handlerThrew, "handler errors should be rethrown");
+    assert(kafka.consumerInstance.commitCalls.length === commitsBeforeThrow, "autoCommit:false handler-error path with no DLQ should NOT commit (kafkajs will redeliver)");
     consumer.shouldThrow = false;
     consumer.shouldRejectAsync = true;
     let asyncHandlerThrew = false;
@@ -128,6 +143,7 @@ const run = async () => {
         asyncHandlerThrew = true;
     }
     assert(asyncHandlerThrew, "async handler rejections should be rethrown");
+    assert(kafka.consumerInstance.commitCalls.length === commitsBeforeThrow, "autoCommit:false async-rejection path with no DLQ should NOT commit");
     consumer.shouldRejectAsync = false;
     consumer.remainingFailures = 2;
     const deadLetterCalls = [];
@@ -195,6 +211,73 @@ const run = async () => {
         deadLetterHookThrowSurfaces = true;
     }
     assert(deadLetterHookThrowSurfaces, "consumer should rethrow when dead-letter hook itself throws");
+    // ─── autoCommit:false interaction with retry + DLQ + commit failures ──
+    const manualCommitConsumer = new TestConsumer(kafka);
+    // Reset commit log so subsequent assertions are independent.
+    kafka.consumerInstance.commitCalls = [];
+    manualCommitConsumer.remainingFailures = 2;
+    await manualCommitConsumer.consume({
+        autoCommit: false,
+        retry: { maxRetries: 3, backoffMs: () => 0 },
+        onDeadLetter: async () => undefined,
+    });
+    await kafka.consumerInstance.eachMessage({
+        message: { value: { toString: () => '{"id":"retry-then-commit"}' }, offset: "10" },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(kafka.consumerInstance.commitCalls.length === 1 &&
+        kafka.consumerInstance.commitCalls[0].offset === "11", "autoCommit:false should commit ONCE after retry-then-success (one commit per message, not per attempt)");
+    // autoCommit:false + retry-exhaustion + DLQ-handled → must commit so the
+    // failed message does not get redelivered forever.
+    kafka.consumerInstance.commitCalls = [];
+    manualCommitConsumer.remainingFailures = 5;
+    let dlqHandled = false;
+    await manualCommitConsumer.consume({
+        autoCommit: false,
+        retry: { maxRetries: 1, backoffMs: () => 0 },
+        onDeadLetter: async () => {
+            dlqHandled = true;
+        },
+    });
+    await kafka.consumerInstance.eachMessage({
+        message: { value: { toString: () => '{"id":"dlq-then-commit"}' }, offset: "20" },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(dlqHandled, "DLQ hook should fire on retry exhaustion");
+    assert(kafka.consumerInstance.commitCalls.length === 1 &&
+        kafka.consumerInstance.commitCalls[0].offset === "21", "autoCommit:false should commit after a DLQ-handled message so it is not redelivered");
+    // commitOffsets failure (rebalance, broker down) MUST be swallowed on the
+    // happy path — at-least-once is preserved by kafkajs replay on the next
+    // session, but propagating would force redelivery + perma-failure loop.
+    kafka.consumerInstance.commitCalls = [];
+    kafka.consumerInstance.commitShouldThrow = true;
+    manualCommitConsumer.remainingFailures = 0; // ensure handler succeeds cleanly
+    let commitFailureSurfaced = false;
+    try {
+        await kafka.consumerInstance.eachMessage({
+            message: { value: { toString: () => '{"id":"commit-fails-but-handler-ok"}' }, offset: "30" },
+            partition: 0,
+            heartbeat: async () => undefined,
+        });
+    }
+    catch (_err) {
+        commitFailureSurfaced = true;
+    }
+    assert(!commitFailureSurfaced, "commitOffsets failure must NOT propagate out of eachMessage (would cause kafkajs to redeliver and never commit)");
+    kafka.consumerInstance.commitShouldThrow = false;
+    // autoCommit:true (or undefined) → kafkajs handles commits, package must NOT
+    // call commitOffsets manually (would double-commit).
+    kafka.consumerInstance.commitCalls = [];
+    const autoCommitConsumer = new TestConsumer(kafka);
+    await autoCommitConsumer.consume({ autoCommit: true });
+    await kafka.consumerInstance.eachMessage({
+        message: { value: { toString: () => '{"id":"auto-commit-no-manual"}' }, offset: "40" },
+        partition: 0,
+        heartbeat: async () => undefined,
+    });
+    assert(kafka.consumerInstance.commitCalls.length === 0, "autoCommit:true must NOT trigger manual commitOffsets (kafkajs owns commit cadence)");
     await consumer.disconnect();
     assert(kafka.consumerInstance.disconnectCalls === 1, "disconnect should use the running consumer instance");
 };

package/dist/consumer.d.ts

@@ -32,10 +32,19 @@ export declare abstract class KafkaConsumer<T extends Event> {
      * - Handler errors are rethrown so kafkajs retries message processing.
      */
     private executeWithRetry;
+    /**
+     * Commit the offset for a single processed message. Used when callers opt
+     * into manual commits via `autoCommit: false`. Failures are logged and
+     * swallowed: at-least-once delivery is preserved because kafkajs will
+     * resume from the last successfully committed offset on the next session.
+     */
+    private commitMessageOffset;
     /**
      * Error handling behavior:
      * - Poison parse errors are skipped so offsets can move forward.
      * - Handler errors are rethrown so kafkajs retries message processing.
+     * - With `autoCommit: false`, offsets are manually committed after a
+     *   message is fully handled (success, poison-skip, or DLQ-handled).
      */
     consume(options?: ConsumeOptions<T["data"]>): Promise<void>;
     parseMessage(msg: Message): T["data"];

package/dist/consumer.js

@@ -53,13 +53,42 @@ class KafkaConsumer {
             }
         }
     }
+    /**
+     * Commit the offset for a single processed message. Used when callers opt
+     * into manual commits via `autoCommit: false`. Failures are logged and
+     * swallowed: at-least-once delivery is preserved because kafkajs will
+     * resume from the last successfully committed offset on the next session.
+     */
+    async commitMessageOffset(consumer, partition, offset) {
+        try {
+            const nextOffset = (BigInt(offset) + BigInt(1)).toString();
+            await consumer.commitOffsets([
+                {
+                    topic: this.topic,
+                    partition,
+                    offset: nextOffset,
+                },
+            ]);
+        }
+        catch (err) {
+            this.logger.error("Failed to commit kafka offset", {
+                topic: this.topic,
+                partition,
+                offset,
+                error: toError(err),
+            });
+        }
+    }
     /**
      * Error handling behavior:
      * - Poison parse errors are skipped so offsets can move forward.
      * - Handler errors are rethrown so kafkajs retries message processing.
+     * - With `autoCommit: false`, offsets are manually committed after a
+     *   message is fully handled (success, poison-skip, or DLQ-handled).
      */
     async consume(options) {
         const consumer = this.getConsumer();
+        const manualCommit = (options === null || options === void 0 ? void 0 : options.autoCommit) === false;
         await consumer.connect();
         await consumer.subscribe({ topics: [this.topic], fromBeginning: true });
         const dlqEnabled = process.env.KAFKA_CONSUMER_DLQ_ENABLED === "true";
@@ -101,6 +130,9 @@ class KafkaConsumer {
                             offset: message.offset,
                             error,
                         });
+                        if (manualCommit) {
+                            await this.commitMessageOffset(consumer, partition, message.offset);
+                        }
                         span.end();
                         return;
                     }
@@ -120,6 +152,9 @@ class KafkaConsumer {
                         if (options === null || options === void 0 ? void 0 : options.onDeadLetter) {
                             try {
                                 await options.onDeadLetter(data, message, error);
+                                if (manualCommit) {
+                                    await this.commitMessageOffset(consumer, partition, message.offset);
+                                }
                                 span.end();
                                 return;
                             }
@@ -135,6 +170,9 @@ class KafkaConsumer {
                         span.end();
                         throw error;
                     }
+                    if (manualCommit) {
+                        await this.commitMessageOffset(consumer, partition, message.offset);
+                    }
                     span.end();
                 });
             },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@taphealth/kafka",
-  "version": "1.6.53",
+  "version": "2.0.0",
   "description": "",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",