kafka-ts 1.1.2-beta.4 → 1.1.3-beta.0

package/dist/consumer/consumer.d.ts

@@ -20,7 +20,9 @@ export type ConsumerOptions = {
     fromBeginning?: boolean;
     fromTimestamp?: bigint;
     retrier?: Retrier;
-    onBatch: (messages: Required<Message>[]) => unknown;
+    onBatch: (messages: Required<Message>[], context: {
+        resolveOffset: (message: Pick<Required<Message>, 'topic' | 'partition' | 'offset'>) => void;
+    }) => unknown;
 };
 export declare class Consumer extends EventEmitter<{
     offsetCommit: [];

package/dist/consumer/consumer.js

@@ -185,7 +185,17 @@ class Consumer extends events_1.default {
         if (!messages.length) {
             return;
         }
-        await retrier(() => options.onBatch(messages));
+        const resolveOffset = (message) => this.offsetManager.resolve(message.topic, message.partition, message.offset + 1n);
+        try {
+            await retrier(() => options.onBatch(messages.filter((message) => !this.offsetManager.isResolved(message)), { resolveOffset }));
+        }
+        catch (error) {
+            await this.consumerGroup
+                ?.offsetCommit(topicPartitions)
+                .then(() => this.offsetManager.flush(topicPartitions))
+                .catch();
+            throw error;
+        }
         response.responses.forEach(({ topicId, partitions }) => {
             partitions.forEach(({ partitionIndex, records }) => {
                 records.forEach(({ baseOffset, lastOffsetDelta }) => {

package/dist/consumer/offset-manager.d.ts

@@ -12,7 +12,13 @@ export declare class OffsetManager {
     pendingOffsets: Record<string, Record<number, bigint>>;
     constructor(options: OffsetManagerOptions);
     getCurrentOffset(topic: string, partition: number): bigint;
+    getPendingOffset(topic: string, partition: number): bigint;
     resolve(topic: string, partition: number, offset: bigint): void;
+    isResolved(message: {
+        topic: string;
+        partition: number;
+        offset: bigint;
+    }): boolean;
     flush(topicPartitions: Record<string, Set<number>>): void;
     fetchOffsets(options: {
         fromTimestamp: bigint;

package/dist/consumer/offset-manager.js

@@ -15,10 +15,17 @@ class OffsetManager {
     getCurrentOffset(topic, partition) {
         return this.currentOffsets[topic]?.[partition] ?? 0n;
     }
+    getPendingOffset(topic, partition) {
+        return this.pendingOffsets[topic]?.[partition] ?? 0n;
+    }
     resolve(topic, partition, offset) {
         this.pendingOffsets[topic] ??= {};
         this.pendingOffsets[topic][partition] = offset;
     }
+    isResolved(message) {
+        return (this.getCurrentOffset(message.topic, message.partition) > message.offset ||
+            this.getPendingOffset(message.topic, message.partition) > message.offset);
+    }
     flush(topicPartitions) {
         Object.entries(topicPartitions).forEach(([topic, partitions]) => {
             this.currentOffsets[topic] ??= {};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.1.2-beta.4",
+    "version": "1.1.3-beta.0",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",