kafka-ts 1.1.2 → 1.1.3

package/dist/api/fetch.js

@@ -4,7 +4,6 @@ exports.FETCH = void 0;
 const api_1 = require("../utils/api");
 const decoder_1 = require("../utils/decoder");
 const error_1 = require("../utils/error");
-const logger_1 = require("../utils/logger");
 exports.FETCH = (0, api_1.createApi)({
     apiKey: 1,
     apiVersion: 15,
@@ -85,8 +84,7 @@ const decodeRecordBatch = (decoder) => {
             continue;
         }
         if (!recordBatchDecoder.canReadBytes(batchLength)) {
-            // likely running into maxBytes limit
-            logger_1.log.debug('Record batch is incomplete, skipping last batch.');
+            // running into maxBytes limit
             recordBatchDecoder.read();
             continue;
         }

package/dist/consumer/consumer.d.ts

@@ -20,7 +20,9 @@ export type ConsumerOptions = {
     fromBeginning?: boolean;
     fromTimestamp?: bigint;
     retrier?: Retrier;
-    onBatch: (messages: Required<Message>[]) => unknown;
+    onBatch: (messages: Required<Message>[], context: {
+        resolveOffset: (message: Pick<Required<Message>, 'topic' | 'partition' | 'offset'>) => void;
+    }) => unknown;
 };
 export declare class Consumer extends EventEmitter<{
     offsetCommit: [];

package/dist/consumer/consumer.js

@@ -185,7 +185,17 @@ class Consumer extends events_1.default {
         if (!messages.length) {
             return;
         }
-        await retrier(() => options.onBatch(messages));
+        const resolveOffset = (message) => this.offsetManager.resolve(message.topic, message.partition, message.offset + 1n);
+        try {
+            await retrier(() => options.onBatch(messages.filter((message) => !this.offsetManager.isResolved(message)), { resolveOffset }));
+        }
+        catch (error) {
+            await this.consumerGroup
+                ?.offsetCommit(topicPartitions)
+                .then(() => this.offsetManager.flush(topicPartitions))
+                .catch();
+            throw error;
+        }
         response.responses.forEach(({ topicId, partitions }) => {
             partitions.forEach(({ partitionIndex, records }) => {
                 records.forEach(({ baseOffset, lastOffsetDelta }) => {

package/dist/consumer/offset-manager.d.ts

@@ -12,7 +12,13 @@ export declare class OffsetManager {
     pendingOffsets: Record<string, Record<number, bigint>>;
     constructor(options: OffsetManagerOptions);
     getCurrentOffset(topic: string, partition: number): bigint;
+    getPendingOffset(topic: string, partition: number): bigint;
     resolve(topic: string, partition: number, offset: bigint): void;
+    isResolved(message: {
+        topic: string;
+        partition: number;
+        offset: bigint;
+    }): boolean;
     flush(topicPartitions: Record<string, Set<number>>): void;
     fetchOffsets(options: {
         fromTimestamp: bigint;

package/dist/consumer/offset-manager.js

@@ -15,10 +15,17 @@ class OffsetManager {
     getCurrentOffset(topic, partition) {
         return this.currentOffsets[topic]?.[partition] ?? 0n;
     }
+    getPendingOffset(topic, partition) {
+        return this.pendingOffsets[topic]?.[partition] ?? 0n;
+    }
     resolve(topic, partition, offset) {
         this.pendingOffsets[topic] ??= {};
         this.pendingOffsets[topic][partition] = offset;
     }
+    isResolved(message) {
+        return (this.getCurrentOffset(message.topic, message.partition) > message.offset ||
+            this.getPendingOffset(message.topic, message.partition) > message.offset);
+    }
     flush(topicPartitions) {
         Object.entries(topicPartitions).forEach(([topic, partitions]) => {
             this.currentOffsets[topic] ??= {};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.1.2",
+    "version": "1.1.3",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",
@@ -23,7 +23,7 @@
         "@types/node": "^20.12.12",
         "prettier": "^3.3.3",
         "typescript": "^5.4.5",
-        "vitest": "^1.6.0"
+        "vitest": "^1.6.1"
     },
     "keywords": [
         "kafka",