kafka-ts 0.0.2-beta → 0.0.2

package/src/connection.ts

@@ -1,14 +1,17 @@
-import assert from "assert";
-import net, { isIP, Socket, TcpSocketConnectOpts } from "net";
-import tls, { TLSSocketOptions } from "tls";
-import { getApiName } from "./api";
-import { Api } from "./utils/api";
-import { Decoder } from "./utils/decoder";
-import { Encoder } from "./utils/encoder";
-import { ConnectionError } from "./utils/error";
-import { trace } from "./utils/tracer";
-
-export type ConnectionOptions = {
+import assert from 'assert';
+import net, { isIP, Socket, TcpSocketConnectOpts } from 'net';
+import tls, { TLSSocketOptions } from 'tls';
+import { getApiName } from './api';
+import { Api } from './utils/api';
+import { Decoder } from './utils/decoder';
+import { Encoder } from './utils/encoder';
+import { ConnectionError } from './utils/error';
+import { log } from './utils/logger';
+import { createTracer } from './utils/tracer';
+
+const trace = createTracer('Connection');
+
+type ConnectionOptions = {
     clientId: string | null;
     connection: TcpSocketConnectOpts;
     ssl: TLSSocketOptions | null;
@@ -22,14 +25,14 @@ export class Connection {
         [correlationId: number]: { resolve: (response: RawResonse) => void; reject: (error: Error) => void };
     } = {};
     private lastCorrelationId = 0;
-    private buffer: Buffer | null = null;
+    private chunks: Buffer[] = [];
 
     constructor(private options: ConnectionOptions) {}
 
     @trace()
     public async connect() {
         this.queue = {};
-        this.buffer = null;
+        this.chunks = [];
 
         await new Promise<void>((resolve, reject) => {
             const { ssl, connection } = this.options;
@@ -44,19 +47,20 @@ export class Connection {
                       resolve,
                   )
                 : net.connect(connection, resolve);
-            this.socket.once("error", reject);
+            this.socket.once('error', reject);
         });
-        this.socket.removeAllListeners("error");
+        this.socket.removeAllListeners('error');
 
-        this.socket.on("data", (data) => this.handleData(data));
-        this.socket.once("close", async () => {
+        this.socket.on('data', (data) => this.handleData(data));
+        this.socket.once('close', async () => {
             Object.values(this.queue).forEach(({ reject }) => {
-                reject(new ConnectionError("Socket closed unexpectedly"));
+                reject(new ConnectionError('Socket closed unexpectedly'));
             });
             this.queue = {};
         });
     }
 
+    @trace()
     public disconnect() {
         this.socket.removeAllListeners();
         return new Promise<void>((resolve) => {
@@ -67,9 +71,10 @@ export class Connection {
         });
     }
 
-    @trace((api, body) => ({ apiName: getApiName(api), body }))
+    @trace((api, body) => ({ message: getApiName(api), body }))
     public async sendRequest<Request, Response>(api: Api<Request, Response>, body: Request): Promise<Response> {
         const correlationId = this.nextCorrelationId();
+        const apiName = getApiName(api);
 
         const encoder = new Encoder()
             .writeInt16(api.apiKey)
@@ -77,18 +82,25 @@ export class Connection {
             .writeInt32(correlationId)
             .writeString(this.options.clientId);
 
-        const request = api.request(encoder, body).value();
-        const requestEncoder = new Encoder().writeInt32(request.length).write(request);
+        const request = api.request(encoder, body);
+        const requestEncoder = new Encoder().writeInt32(request.getByteLength()).writeEncoder(request);
 
+        let timeout: NodeJS.Timeout | undefined;
         const { responseDecoder, responseSize } = await new Promise<RawResonse>(async (resolve, reject) => {
+            timeout = setTimeout(() => {
+                delete this.queue[correlationId];
+                reject(new ConnectionError(`${apiName} timed out`));
+            }, 30_000);
+
             try {
-                await this.write(requestEncoder.value());
                 this.queue[correlationId] = { resolve, reject };
+                await this.write(requestEncoder.value());
             } catch (error) {
                 reject(error);
             }
         });
-        const response = api.response(responseDecoder);
+        clearTimeout(timeout);
+        const response = await api.response(responseDecoder);
 
         assert(
             responseDecoder.getOffset() - 4 === responseSize,
@@ -100,7 +112,7 @@ export class Connection {
 
     private write(buffer: Buffer) {
         return new Promise<void>((resolve, reject) => {
-            const { stack } = new Error("Write error");
+            const { stack } = new Error('Write error');
             this.socket.write(buffer, (error) => {
                 if (error) {
                     const err = new ConnectionError(error.message);
@@ -113,12 +125,13 @@ export class Connection {
     }
 
     private handleData(buffer: Buffer) {
-        this.buffer = this.buffer ? Buffer.concat([this.buffer, buffer]) : buffer;
-        if (this.buffer.length < 4) {
+        this.chunks.push(buffer);
+
+        const decoder = new Decoder(Buffer.concat(this.chunks));
+        if (decoder.getBufferLength() < 4) {
             return;
         }
 
-        const decoder = new Decoder(this.buffer);
         const size = decoder.readInt32();
         if (size !== decoder.getBufferLength() - 4) {
             return;
@@ -126,15 +139,18 @@ export class Connection {
 
         const correlationId = decoder.readInt32();
 
-        const { resolve } = this.queue[correlationId];
-        delete this.queue[correlationId];
-
-        resolve({ responseDecoder: decoder, responseSize: size });
-        this.buffer = null;
+        const context = this.queue[correlationId];
+        if (context) {
+            delete this.queue[correlationId];
+            context.resolve({ responseDecoder: decoder, responseSize: size });
+        } else {
+            log.debug('Could not find pending request for correlationId', { correlationId });
+        }
+        this.chunks = [];
     }
 
     private nextCorrelationId() {
-        return (this.lastCorrelationId = (this.lastCorrelationId + 1) % 2 ** 31);
+        return this.lastCorrelationId++;
     }
 }
 

package/src/consumer/consumer-group.ts

@@ -1,10 +1,14 @@
-import { API, API_ERROR } from "../api";
-import { KEY_TYPE } from "../api/find-coordinator";
-import { Assignment, MemberAssignment } from "../api/sync-group";
-import { Cluster } from "../cluster";
-import { KafkaTSApiError, KafkaTSError } from "../utils/error";
-import { ConsumerMetadata } from "./consumer-metadata";
-import { OffsetManager } from "./offset-manager";
+import EventEmitter from 'events';
+import { API, API_ERROR } from '../api';
+import { KEY_TYPE } from '../api/find-coordinator';
+import { Assignment, MemberAssignment } from '../api/sync-group';
+import { Cluster } from '../cluster';
+import { KafkaTSApiError, KafkaTSError } from '../utils/error';
+import { createTracer } from '../utils/tracer';
+import { ConsumerMetadata } from './consumer-metadata';
+import { OffsetManager } from './offset-manager';
+
+const trace = createTracer('ConsumerGroup');
 
 type ConsumerGroupOptions = {
     cluster: Cluster;
@@ -17,19 +21,25 @@ type ConsumerGroupOptions = {
     offsetManager: OffsetManager;
 };
 
-export class ConsumerGroup {
+export class ConsumerGroup extends EventEmitter<{ offsetCommit: [] }> {
     private coordinatorId = -1;
-    private memberId = "";
+    private memberId = '';
     private generationId = -1;
-    private leaderId = "";
+    private leaderId = '';
     private memberIds: string[] = [];
     private heartbeatInterval: NodeJS.Timeout | null = null;
     private heartbeatError: KafkaTSError | null = null;
 
-    constructor(private options: ConsumerGroupOptions) {}
+    constructor(private options: ConsumerGroupOptions) {
+        super();
+    }
 
+    @trace()
     public async join() {
         await this.findCoordinator();
+        await this.options.cluster.setSeedBroker(this.coordinatorId);
+
+        this.memberId = '';
         await this.joinGroup();
         await this.syncGroup();
         await this.offsetFetch();
@@ -53,12 +63,16 @@ export class ConsumerGroup {
         }
     }
 
-    public async handleLastHeartbeat() {
+    public handleLastHeartbeat() {
         if (this.heartbeatError) {
             throw this.heartbeatError;
         }
     }
 
+    public resetHeartbeat() {
+        this.heartbeatError = null;
+    }
+
     private async findCoordinator() {
         const { coordinators } = await this.options.cluster.sendRequest(API.FIND_COORDINATOR, {
             keyType: KEY_TYPE.GROUP,
@@ -70,14 +84,14 @@ export class ConsumerGroup {
     private async joinGroup(): Promise<void> {
         const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
         try {
-            const response = await cluster.sendRequestToNode(this.coordinatorId)(API.JOIN_GROUP, {
+            const response = await cluster.sendRequest(API.JOIN_GROUP, {
                 groupId,
                 groupInstanceId,
                 memberId: this.memberId,
                 sessionTimeoutMs,
                 rebalanceTimeoutMs,
-                protocolType: "consumer",
-                protocols: [{ name: "RoundRobinAssigner", metadata: { version: 0, topics } }],
+                protocolType: 'consumer',
+                protocols: [{ name: 'RoundRobinAssigner', metadata: { version: 0, topics } }],
                 reason: null,
             });
             this.memberId = response.memberId;
@@ -113,16 +127,16 @@ export class ConsumerGroup {
             assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
         }
 
-        const response = await cluster.sendRequestToNode(this.coordinatorId)(API.SYNC_GROUP, {
+        const response = await cluster.sendRequest(API.SYNC_GROUP, {
             groupId,
             groupInstanceId,
             memberId: this.memberId,
             generationId: this.generationId,
-            protocolType: "consumer",
-            protocolName: "RoundRobinAssigner",
+            protocolType: 'consumer',
+            protocolName: 'RoundRobinAssigner',
             assignments,
         });
-        metadata.setAssignment(JSON.parse(response.assignments || "{}") as Assignment);
+        metadata.setAssignment(JSON.parse(response.assignments || '{}') as Assignment);
     }
 
     private async offsetFetch() {
@@ -144,31 +158,35 @@ export class ConsumerGroup {
         };
         if (!request.groups.length) return;
 
-        const response = await cluster.sendRequestToNode(this.coordinatorId)(API.OFFSET_FETCH, request);
+        const response = await cluster.sendRequest(API.OFFSET_FETCH, request);
+
+        const topicPartitions: Record<string, Set<number>> = {};
         response.groups.forEach((group) => {
             group.topics.forEach((topic) => {
-                topic.partitions
-                    .filter(({ committedOffset }) => committedOffset >= 0)
-                    .forEach(({ partitionIndex, committedOffset }) =>
-                        offsetManager.resolve(topic.name, partitionIndex, committedOffset),
-                    );
+                topicPartitions[topic.name] ??= new Set();
+                topic.partitions.forEach(({ partitionIndex, committedOffset }) => {
+                    if (committedOffset >= 0) {
+                        topicPartitions[topic.name].add(partitionIndex);
+                        offsetManager.resolve(topic.name, partitionIndex, committedOffset);
+                    }
+                });
             });
         });
-        offsetManager.flush();
+        offsetManager.flush(topicPartitions);
     }
 
-    public async offsetCommit() {
+    public async offsetCommit(topicPartitions: Record<string, Set<number>>) {
         const { cluster, groupId, groupInstanceId, offsetManager } = this.options;
         const request = {
             groupId,
             groupInstanceId,
             memberId: this.memberId,
             generationIdOrMemberEpoch: this.generationId,
-            topics: Object.entries(offsetManager.pendingOffsets).map(([topic, partitions]) => ({
+            topics: Object.entries(topicPartitions).map(([topic, partitions]) => ({
                 name: topic,
-                partitions: Object.entries(partitions).map(([partition, offset]) => ({
-                    partitionIndex: parseInt(partition),
-                    committedOffset: offset,
+                partitions: [...partitions].map((partitionIndex) => ({
+                    partitionIndex,
+                    committedOffset: offsetManager.pendingOffsets[topic][partitionIndex],
                     committedLeaderEpoch: -1,
                     committedMetadata: null,
                 })),
@@ -177,13 +195,13 @@ export class ConsumerGroup {
         if (!request.topics.length) {
             return;
         }
-        await cluster.sendRequestToNode(this.coordinatorId)(API.OFFSET_COMMIT, request);
-        offsetManager.flush();
+        await cluster.sendRequest(API.OFFSET_COMMIT, request);
+        this.emit('offsetCommit');
     }
 
     public async heartbeat() {
         const { cluster, groupId, groupInstanceId } = this.options;
-        await cluster.sendRequestToNode(this.coordinatorId)(API.HEARTBEAT, {
+        await cluster.sendRequest(API.HEARTBEAT, {
             groupId,
             groupInstanceId,
             memberId: this.memberId,
@@ -192,10 +210,14 @@ export class ConsumerGroup {
     }
 
     public async leaveGroup() {
+        if (this.coordinatorId === -1) {
+            return;
+        }
+
         const { cluster, groupId, groupInstanceId } = this.options;
         this.stopHeartbeater();
         try {
-            await cluster.sendRequestToNode(this.coordinatorId)(API.LEAVE_GROUP, {
+            await cluster.sendRequest(API.LEAVE_GROUP, {
                 groupId,
                 members: [{ memberId: this.memberId, groupInstanceId, reason: null }],
             });

package/src/consumer/consumer-metadata.ts

@@ -1,5 +1,5 @@
-import { Assignment } from "../api/sync-group";
-import { Metadata } from "../metadata";
+import { Assignment } from '../api/sync-group';
+import { Metadata } from '../metadata';
 
 export class ConsumerMetadata extends Metadata {
     private assignment: Assignment = {};

package/src/consumer/consumer.ts

@@ -1,15 +1,20 @@
-import { API, API_ERROR } from "../api";
-import { IsolationLevel } from "../api/fetch";
-import { Assignment } from "../api/sync-group";
-import { Cluster } from "../cluster";
-import { distributeAssignmentsToNodes } from "../distributors/assignments-to-replicas";
-import { Message } from "../types";
-import { delay } from "../utils/delay";
-import { ConnectionError, KafkaTSApiError } from "../utils/error";
-import { defaultRetrier, Retrier } from "../utils/retrier";
-import { ConsumerGroup } from "./consumer-group";
-import { ConsumerMetadata } from "./consumer-metadata";
-import { OffsetManager } from "./offset-manager";
+import EventEmitter from 'events';
+import { API, API_ERROR } from '../api';
+import { IsolationLevel } from '../api/fetch';
+import { Assignment } from '../api/sync-group';
+import { Cluster } from '../cluster';
+import { distributeMessagesToTopicPartitionLeaders } from '../distributors/messages-to-topic-partition-leaders';
+import { Message } from '../types';
+import { delay } from '../utils/delay';
+import { ConnectionError, KafkaTSApiError } from '../utils/error';
+import { log } from '../utils/logger';
+import { createTracer } from '../utils/tracer';
+import { ConsumerGroup } from './consumer-group';
+import { ConsumerMetadata } from './consumer-metadata';
+import { BatchGranularity, FetchManager } from './fetch-manager';
+import { OffsetManager } from './offset-manager';
+
+const trace = createTracer('Consumer');
 
 export type ConsumerOptions = {
     topics: string[];
@@ -25,35 +30,40 @@ export type ConsumerOptions = {
     partitionMaxBytes?: number;
     allowTopicAutoCreation?: boolean;
     fromBeginning?: boolean;
-    retrier?: Retrier;
-} & ({ onMessage: (message: Message) => unknown } | { onBatch: (messages: Message[]) => unknown });
+    batchGranularity?: BatchGranularity;
+    concurrency?: number;
+} & ({ onBatch: (messages: Required<Message>[]) => unknown } | { onMessage: (message: Required<Message>) => unknown });
 
-export class Consumer {
+export class Consumer extends EventEmitter<{ offsetCommit: [] }> {
     private options: Required<ConsumerOptions>;
     private metadata: ConsumerMetadata;
     private consumerGroup: ConsumerGroup | undefined;
     private offsetManager: OffsetManager;
+    private fetchManager?: FetchManager;
     private stopHook: (() => void) | undefined;
 
     constructor(
         private cluster: Cluster,
         options: ConsumerOptions,
     ) {
+        super();
+
         this.options = {
             ...options,
             groupId: options.groupId ?? null,
             groupInstanceId: options.groupInstanceId ?? null,
-            rackId: options.rackId ?? "",
+            rackId: options.rackId ?? '',
             sessionTimeoutMs: options.sessionTimeoutMs ?? 30_000,
             rebalanceTimeoutMs: options.rebalanceTimeoutMs ?? 60_000,
             maxWaitMs: options.maxWaitMs ?? 5000,
             minBytes: options.minBytes ?? 1,
-            maxBytes: options.maxBytes ?? 1_000_000,
-            partitionMaxBytes: options.partitionMaxBytes ?? 1_000_000,
+            maxBytes: options.maxBytes ?? 1_048_576,
+            partitionMaxBytes: options.partitionMaxBytes ?? 1_048_576,
             isolationLevel: options.isolationLevel ?? IsolationLevel.READ_UNCOMMITTED,
             allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
             fromBeginning: options.fromBeginning ?? false,
-            retrier: options.retrier ?? defaultRetrier,
+            batchGranularity: options.batchGranularity ?? 'partition',
+            concurrency: options.concurrency ?? 1,
         };
 
         this.metadata = new ConsumerMetadata({ cluster: this.cluster });
@@ -74,8 +84,10 @@ export class Consumer {
                   offsetManager: this.offsetManager,
               })
             : undefined;
+        this.consumerGroup?.on('offsetCommit', () => this.emit('offsetCommit'));
     }
 
+    @trace()
     public async start(): Promise<void> {
         const { topics, allowTopicAutoCreation, fromBeginning } = this.options;
 
@@ -88,88 +100,82 @@ export class Consumer {
             await this.offsetManager.fetchOffsets({ fromBeginning });
             await this.consumerGroup?.join();
         } catch (error) {
-            console.error(error);
-            console.debug(`Restarting consumer in 1 second...`);
+            log.warn('Failed to start consumer', error);
+            log.debug(`Restarting consumer in 1 second...`);
             await delay(1000);
 
             if (this.stopHook) return (this.stopHook as () => void)();
             return this.close(true).then(() => this.start());
         }
-        this.fetchLoop();
+        this.startFetchManager();
     }
 
-    private fetchLoop = async () => {
-        const { options } = this;
-        const { retrier } = options;
+    @trace()
+    public async close(force = false): Promise<void> {
+        if (!force) {
+            await new Promise<void>(async (resolve) => {
+                this.stopHook = resolve;
+                await this.fetchManager?.stop();
+            });
+        }
+        await this.consumerGroup?.leaveGroup().catch((error) => log.debug(`Failed to leave group: ${error.message}`));
+        await this.cluster.disconnect().catch((error) => log.debug(`Failed to disconnect: ${error.message}`));
+    }
 
-        let nodeAssignments: { nodeId: number; assignment: Assignment }[] = [];
-        let shouldReassign = true;
+    private async startFetchManager() {
+        const { batchGranularity, concurrency } = this.options;
 
         while (!this.stopHook) {
-            if (shouldReassign || !nodeAssignments) {
-                nodeAssignments = Object.entries(
-                    distributeAssignmentsToNodes(
-                        this.metadata.getAssignment(),
-                        this.metadata.getTopicPartitionReplicaIds(),
+            this.consumerGroup?.resetHeartbeat();
+
+            // TODO: If leader is not available, find another read replica
+            const nodeAssignments = Object.entries(
+                distributeMessagesToTopicPartitionLeaders(
+                    Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) =>
+                        partitions.map((partition) => ({ topic, partition })),
                     ),
-                ).map(([nodeId, assignment]) => ({ nodeId: parseInt(nodeId), assignment }));
-                shouldReassign = false;
-            }
+                    this.metadata.getTopicPartitionLeaderIds(),
+                ),
+            ).map(([nodeId, assignment]) => ({
+                nodeId: parseInt(nodeId),
+                assignment: Object.fromEntries(
+                    Object.entries(assignment).map(([topic, partitions]) => [
+                        topic,
+                        Object.keys(partitions).map(Number),
+                    ]),
+                ),
+            }));
+
+            const numPartitions = Object.values(this.metadata.getAssignment()).flat().length;
+            const numProcessors = Math.min(concurrency, numPartitions);
+
+            this.fetchManager = new FetchManager({
+                fetch: this.fetch.bind(this),
+                process: this.process.bind(this),
+                metadata: this.metadata,
+                consumerGroup: this.consumerGroup,
+                nodeAssignments,
+                batchGranularity,
+                concurrency: numProcessors,
+            });
 
             try {
-                for (const { nodeId, assignment } of nodeAssignments) {
-                    const batch = await this.fetch(nodeId, assignment);
-                    const messages = batch.responses.flatMap(({ topicId, partitions }) =>
-                        partitions.flatMap(({ partitionIndex, records }) =>
-                            records.flatMap(({ baseTimestamp, baseOffset, records }) =>
-                                records.map(
-                                    (message): Required<Message> => ({
-                                        topic: this.metadata.getTopicNameById(topicId),
-                                        partition: partitionIndex,
-                                        key: message.key ?? null,
-                                        value: message.value ?? null,
-                                        headers: Object.fromEntries(
-                                            message.headers.map(({ key, value }) => [key, value]),
-                                        ),
-                                        timestamp: baseTimestamp + BigInt(message.timestampDelta),
-                                        offset: baseOffset + BigInt(message.offsetDelta),
-                                    }),
-                                ),
-                            ),
-                        ),
-                    );
-
-                    if ("onBatch" in options) {
-                        await retrier(() => options.onBatch(messages));
-
-                        messages.forEach(({ topic, partition, offset }) =>
-                            this.offsetManager.resolve(topic, partition, offset + 1n),
-                        );
-                    } else if ("onMessage" in options) {
-                        for (const message of messages) {
-                            await retrier(() => options.onMessage(message));
-
-                            const { topic, partition, offset } = message;
-                            this.offsetManager.resolve(topic, partition, offset + 1n);
-                        }
-                    }
-                    await this.consumerGroup?.offsetCommit();
-                    await this.consumerGroup?.handleLastHeartbeat();
-                }
+                await this.fetchManager.start();
 
                 if (!nodeAssignments.length) {
-                    console.debug("No partitions assigned. Waiting for reassignment...");
+                    log.debug('No partitions assigned. Waiting for reassignment...');
                     await delay(this.options.maxWaitMs);
-                    await this.consumerGroup?.handleLastHeartbeat();
+                    this.consumerGroup?.handleLastHeartbeat();
                 }
             } catch (error) {
+                await this.fetchManager.stop();
+
                 if ((error as KafkaTSApiError).errorCode === API_ERROR.REBALANCE_IN_PROGRESS) {
-                    console.debug("Rebalance in progress...");
-                    shouldReassign = true;
+                    log.debug('Rebalance in progress...');
                     continue;
                 }
                 if ((error as KafkaTSApiError).errorCode === API_ERROR.FENCED_INSTANCE_ID) {
-                    console.debug("New consumer with the same groupInstanceId joined. Exiting the consumer...");
+                    log.debug('New consumer with the same groupInstanceId joined. Exiting the consumer...');
                     this.close();
                     break;
                 }
@@ -177,28 +183,45 @@ export class Consumer {
                     error instanceof ConnectionError ||
                     (error instanceof KafkaTSApiError && error.errorCode === API_ERROR.NOT_COORDINATOR)
                 ) {
-                    console.debug(`${error.message}. Restarting consumer...`);
+                    log.debug(`${error.message}. Restarting consumer...`);
                     this.close().then(() => this.start());
                     break;
                 }
-                console.error(error);
-                await this.consumerGroup?.offsetCommit();
+                log.error((error as Error).message, error);
+                this.close();
                 break;
             }
         }
         this.stopHook?.();
-    };
+    }
 
-    public async close(force = false): Promise<void> {
-        if (!force) {
-            await new Promise<void>((resolve) => {
-                this.stopHook = resolve;
-            });
+    @trace((messages) => ({ count: messages.length }))
+    private async process(messages: Required<Message>[]) {
+        const { options } = this;
+
+        const topicPartitions: Record<string, Set<number>> = {};
+        for (const { topic, partition } of messages) {
+            topicPartitions[topic] ??= new Set();
+            topicPartitions[topic].add(partition);
         }
-        await this.consumerGroup
-            ?.leaveGroup()
-            .catch((error) => console.warn(`Failed to leave group: ${error.message}`));
-        await this.cluster.disconnect().catch((error) => console.warn(`Failed to disconnect: ${error.message}`));
+
+        if ('onBatch' in options) {
+            await options.onBatch(messages);
+
+            messages.forEach(({ topic, partition, offset }) =>
+                this.offsetManager.resolve(topic, partition, offset + 1n),
+            );
+        } else if ('onMessage' in options) {
+            for (const message of messages) {
+                await options.onMessage(message);
+
+                const { topic, partition, offset } = message;
+                this.offsetManager.resolve(topic, partition, offset + 1n);
+            }
+        }
+
+        await this.consumerGroup?.offsetCommit(topicPartitions);
+        this.offsetManager.flush(topicPartitions);
     }
 
     private fetch(nodeId: number, assignment: Assignment) {