@platformatic/kafka 2.0.1 → 2.2.2

package/dist/clients/base/base.d.ts

@@ -84,11 +84,11 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
     clearMetadata(): void;
-    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
     [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
     [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
     [kGetConnection](broker: Broker, callback: Callback<Connection>): void;
-    [kGetBootstrapConnection](callback: Callback<Connection>): void;
+    [kGetBootstrapConnection](callback: Callback<Connection>, attempt?: number): void;
     [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
     [kInspect](...args: unknown[]): void;
     [kFormatValidationErrors](validator: ValidateFunction<unknown>, targetName: string): string;

package/dist/clients/base/base.js

@@ -55,7 +55,12 @@ export class Base extends TypedEventEmitter {
         this[kApis] = [];
         this[kContext] = options.context;
         // Validate options
-        this[kOptions] = Object.assign({}, defaultBaseOptions, options);
+        this[kOptions] = Object.assign({}, defaultBaseOptions);
+        for (const [key, value] of Object.entries(options)) {
+            if (value !== undefined) {
+                this[kOptions][key] = value;
+            }
+        }
         this[kValidateOptions](this[kOptions], baseOptionsValidator, '/options');
         this[kClientId] = options.clientId;
         if (typeof this[kOptions].retries === 'boolean') {
@@ -201,7 +206,7 @@ export class Base extends TypedEventEmitter {
     }
     [kListApis](callback) {
         this[kPerformDeduplicated]('listApis', deduplicateCallback => {
-            this[kPerformWithRetry]('listApis', retryCallback => {
+            this[kPerformWithRetry]('listApis', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
@@ -209,7 +214,7 @@ export class Base extends TypedEventEmitter {
                     }
                     // We use V3 to be able to get APIS from Kafka 2.4.0+
                     apiVersionsV3(connection, clientSoftwareName, clientSoftwareVersion, retryCallback);
-                });
+                }, attempt);
             }, (error, metadata) => {
                 if (error) {
                     deduplicateCallback(error);
@@ -255,7 +260,13 @@ export class Base extends TypedEventEmitter {
                     this.emitWithDebug('client', 'performWithRetry:retry', operationId, attempt, retries, delay);
                     const timeout = setTimeout(() => {
                         this.removeListener('client:close', onClose);
-                        this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                        try {
+                            this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                        }
+                        catch (error) {
+                            errors.push(error);
+                            callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors));
+                        }
                     }, delay);
                     this.once('client:close', onClose);
                 }
@@ -274,7 +285,7 @@ export class Base extends TypedEventEmitter {
                 errors.splice(0, errors.length);
             }
             callback(null, result);
-        });
+        }, attempt);
         return callback[kCallbackPromise];
     }
     [kPerformDeduplicated](operationId, operation, callback) {
@@ -329,13 +340,20 @@ export class Base extends TypedEventEmitter {
     [kGetConnection](broker, callback) {
         this[kConnections].get(broker, callback);
     }
-    [kGetBootstrapConnection](callback) {
+    [kGetBootstrapConnection](callback, attempt = 0) {
+        let brokers;
         if (!this.#metadata) {
-            this[kConnections].getFirstAvailable(this[kBootstrapBrokers], callback);
-            return;
+            brokers = this[kBootstrapBrokers];
+        }
+        else {
+            const discovered = Array.from(this.#metadata.brokers.values());
+            brokers = [...this[kBootstrapBrokers], ...discovered];
         }
-        const discovered = Array.from(this.#metadata.brokers.values());
-        this[kConnections].getFirstAvailable([...this[kBootstrapBrokers], ...discovered], callback);
+        if (attempt > 0 && brokers.length > 1) {
+            const offset = attempt % brokers.length;
+            brokers = [...brokers.slice(offset), ...brokers.slice(0, offset)];
+        }
+        this[kConnections].getFirstAvailable(brokers, callback);
     }
     [kValidateOptions](target, validator, targetName, throwOnErrors = true) {
         if (!this[kOptions].strict) {
@@ -390,7 +408,7 @@ export class Base extends TypedEventEmitter {
         this[kPerformDeduplicated](
         // Unique key to avoid mixing callbacks
         `metadata-${topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
-            this[kPerformWithRetry]('metadata', retryCallback => {
+            this[kPerformWithRetry]('metadata', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
@@ -403,7 +421,7 @@ export class Base extends TypedEventEmitter {
                         }
                         api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
                     });
-                });
+                }, attempt);
             }, (error, metadata) => {
                 if (error) {
                     const unknownTopicError = error.findBy('apiCode', 3);

package/dist/clients/base/options.d.ts

@@ -25,6 +25,9 @@ export declare const baseOptionsSchema: {
             type: string;
             pattern: string;
         };
+        clientRack: {
+            type: string;
+        };
         bootstrapBrokers: {
             oneOf: ({
                 type: string;

package/dist/clients/base/options.js

@@ -14,6 +14,7 @@ export const baseOptionsSchema = {
     type: 'object',
     properties: {
         clientId: idProperty,
+        clientRack: { type: 'string' },
         bootstrapBrokers: {
             oneOf: [
                 { type: 'array', items: { type: 'string' } },

package/dist/clients/base/types.d.ts

@@ -32,6 +32,7 @@ export interface ClusterMetadata {
 export type RetryDelayGetter<Owner = object> = (client: Owner, operationId: string, attempt: number, retries: number, error: Error) => number;
 export interface BaseOptions extends ConnectionOptions {
     clientId: string;
+    clientRack?: string;
     bootstrapBrokers: Broker[] | string[];
     context?: unknown;
     timeout?: number;

package/dist/clients/consumer/consumer.d.ts

@@ -1,7 +1,9 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type FetchResponse } from '../../apis/consumer/fetch-v17.ts';
 import { type ConnectionPool } from '../../network/connection-pool.ts';
+import { kGetFetchNode } from '../../symbols.ts';
 import { Base, type BaseEvents, kCreateConnectionPool } from '../base/base.ts';
+import { type ClusterMetadata } from '../base/types.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
 import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
@@ -56,4 +58,5 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     joinGroup(options?: GroupOptions): Promise<string>;
     leaveGroup(force?: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     leaveGroup(force?: boolean): Promise<void>;
+    [kGetFetchNode](metadata: ClusterMetadata, topic: string, partition: number, now: number): number;
 }

package/dist/clients/consumer/consumer.js

@@ -7,7 +7,7 @@ import { INT32_SIZE } from "../../protocol/definitions.js";
 import { Reader } from "../../protocol/reader.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
-import { kAutocommit, kRefreshOffsetsAndFetch } from "../../symbols.js";
+import { kAutocommit, kGetFetchNode, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { emitExperimentalApiWarning } from "../../utils.js";
 import { Base, kAfterCreate, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
@@ -15,6 +15,7 @@ import { MessagesStream } from "./messages-stream.js";
 import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
 import { roundRobinAssigner } from "./partitions-assigners.js";
 import { TopicsMap } from "./topics-map.js";
+import { partitionKey } from "./utils.js";
 export class Consumer extends Base {
     groupId;
     groupInstanceId;
@@ -34,6 +35,8 @@ export class Consumer extends Base {
     #useConsumerGroupProtocol;
     #memberEpoch;
     #groupRemoteAssignor;
+    #clientRack;
+    #preferredReadReplicas;
     #streams;
     #lagMonitoring;
     #streamContext;
@@ -78,6 +81,8 @@ export class Consumer extends Base {
         this.#memberEpoch = 0;
         this.#useConsumerGroupProtocol = this[kOptions].groupProtocol === 'consumer';
         this.#groupRemoteAssignor = this[kOptions].groupRemoteAssignor ?? null;
+        this.#clientRack = this[kOptions].clientRack ?? '';
+        this.#preferredReadReplicas = new Map();
         this.#streamContext = options.streamContext ?? options.context;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
         if (this[kPrometheus]) {
@@ -354,7 +359,7 @@ export class Consumer extends Base {
         if (this[kCheckNotClosed](callback)) {
             return callback[kCallbackPromise];
         }
-        if (this.#coordinatorId) {
+        if (this.#coordinatorId !== null) {
             callback(null, this.#coordinatorId);
             return callback[kCallbackPromise];
         }
@@ -422,6 +427,52 @@ export class Consumer extends Base {
     #consume(options, callback) {
         consumerConsumesChannel.traceCallback(this.#performConsume, 2, createDiagnosticContext({ client: this, operation: 'consume', options }), this, options, true, callback);
     }
+    [kGetFetchNode](metadata, topic, partition, now) {
+        const partitionMetadata = metadata.topics.get(topic).partitions[partition];
+        const key = partitionKey(topic, partition);
+        const preferredReadReplica = this.#preferredReadReplicas.get(key);
+        if (preferredReadReplica === undefined) {
+            return partitionMetadata.leader;
+        }
+        if (now > preferredReadReplica.expiresAt) {
+            this.#preferredReadReplicas.delete(key);
+            return partitionMetadata.leader;
+        }
+        if (metadata.brokers.has(preferredReadReplica.node) &&
+            partitionMetadata.replicas.includes(preferredReadReplica.node) &&
+            !partitionMetadata.offlineReplicas.includes(preferredReadReplica.node)) {
+            return preferredReadReplica.node;
+        }
+        this.#preferredReadReplicas.delete(key);
+        this.clearMetadata();
+        return partitionMetadata.leader;
+    }
+    #topicIdsById(metadata) {
+        const topicIds = new Map();
+        for (const [topic, { id }] of metadata.topics) {
+            topicIds.set(id, topic);
+        }
+        return topicIds;
+    }
+    #fetchNodeForRequest(metadata, fallbackNode, topics, topicIds, now) {
+        let requestNode;
+        for (const topicRequest of topics) {
+            const topic = topicIds.get(topicRequest.topicId);
+            if (!topic) {
+                return fallbackNode;
+            }
+            for (const { partition } of topicRequest.partitions) {
+                const node = this[kGetFetchNode](metadata, topic, partition, now);
+                if (requestNode === undefined) {
+                    requestNode = node;
+                }
+                else if (requestNode !== node) {
+                    return fallbackNode;
+                }
+            }
+        }
+        return requestNode ?? fallbackNode;
+    }
     #fetch(options, callback) {
         const isolationLevel = options.isolationLevel ?? this[kOptions].isolationLevel;
         this[kPerformWithRetry]('fetch', retryCallback => {
@@ -430,14 +481,17 @@ export class Consumer extends Base {
                     retryCallback(error);
                     return;
                 }
-                const broker = metadata.brokers.get(options.node);
+                const topicIds = this.#topicIdsById(metadata);
+                const node = this.#fetchNodeForRequest(metadata, options.node, options.topics, topicIds, Date.now());
+                const broker = metadata.brokers.get(node);
                 if (!broker) {
-                    retryCallback(new UserError(`Cannot find broker with node id ${options.node}`));
+                    retryCallback(new UserError(`Cannot find broker with node id ${node}`));
                     return;
                 }
                 const pool = options.connectionPool ?? this[kConnections];
                 pool.get(broker, (error, connection) => {
                     if (error) {
+                        this.#clearPreferredReadReplicas(options.topics, topicIds);
                         // When a connection was not available (either interrupted or not available) we
                         // reset the leader epoch in the options so that when connection is re-established again we can continue
                         for (const topic of options.topics) {
@@ -454,8 +508,9 @@ export class Consumer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, 0, 0, options.topics, [], '', (error, result) => {
+                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, 0, 0, options.topics, [], this.#clientRack, (error, result) => {
                             if (error) {
+                                this.#clearPreferredReadReplicas(options.topics, topicIds);
                                 const genericError = error;
                                 if (genericError.findBy?.('apiId', 'FENCED_LEADER_EPOCH')) {
                                     this.clearMetadata();
@@ -467,6 +522,9 @@ export class Consumer extends Base {
                                     }
                                 }
                             }
+                            else {
+                                this.#updatePreferredReadReplicas(metadata, topicIds, result);
+                            }
                             retryCallback(error, result);
                         });
                     });
@@ -581,7 +639,7 @@ export class Consumer extends Base {
                 }, concurrentCallback, 0);
             }, (error, responses) => {
                 if (error) {
-                    callback(this.#handleMetadataError(error));
+                    callback(this.#handleError(error));
                     return;
                 }
                 let offsets = new Map();
@@ -646,14 +704,14 @@ export class Consumer extends Base {
                     error.findBy?.('apiId', 'STALE_MEMBER_EPOCH')) {
                     this.#consumerGroupHeartbeat(this[kOptions], heartbeatError => {
                         if (heartbeatError) {
-                            callback(this.#handleMetadataError(heartbeatError));
+                            callback(this.#handleError(heartbeatError));
                             return;
                         }
                         this.#listCommittedOffsets(options, callback, staleMemberEpochRetries + 1);
                     });
                     return;
                 }
-                callback(this.#handleMetadataError(error));
+                callback(this.#handleError(error));
                 return;
             }
             const committed = new Map();
@@ -671,7 +729,7 @@ export class Consumer extends Base {
         });
     }
     #findGroupCoordinator(callback) {
-        if (this.#coordinatorId) {
+        if (this.#coordinatorId !== null) {
             callback(null, this.#coordinatorId);
             return;
         }
@@ -748,8 +806,7 @@ export class Consumer extends Base {
                     return;
                 }
                 const memberId = this.#getConsumerGroupHeartbeatMemberId(api.version);
-                api(connection, this.groupId, memberId, this.#memberEpoch, this.groupInstanceId, null, // rackId
-                options.rebalanceTimeout, this.topics.current, null, this.#groupRemoteAssignor, this.#assignments, groupCallback);
+                api(connection, this.groupId, memberId, this.#memberEpoch, this.groupInstanceId, this.#clientRack || null, options.rebalanceTimeout, this.topics.current, null, this.#groupRemoteAssignor, this.#assignments, groupCallback);
             });
         }, (error, response) => {
             if (this[kClosed]) {
@@ -764,6 +821,7 @@ export class Consumer extends Base {
                 if (fenced) {
                     this.#assignments = [];
                     this.assignments = [];
+                    this.#syncPreferredReadReplicas();
                     this.#memberEpoch = 0;
                     this.#consumerGroupHeartbeat(options, () => { });
                     callback(error);
@@ -878,6 +936,7 @@ export class Consumer extends Base {
             }));
             this.#assignments = newAssignments;
             this.assignments = assignments;
+            this.#syncPreferredReadReplicas();
             callback(null);
         });
     }
@@ -903,8 +962,7 @@ export class Consumer extends Base {
                 }
                 const memberId = this.#getConsumerGroupHeartbeatMemberId(api.version);
                 api(connection, this.groupId, memberId, -1, // memberEpoch = -1 signals leave
-                this.groupInstanceId, null, // rackId
-                0, // rebalanceTimeout
+                this.groupInstanceId, this.#clientRack || null, 0, // rebalanceTimeout
                 [], // subscribedTopicNames
                 null, // subscribedTopicRegex
                 this.#groupRemoteAssignor, [], // topicPartitions
@@ -916,6 +974,7 @@ export class Consumer extends Base {
             this.#memberEpoch = -1;
             this.#assignments = [];
             this.assignments = [];
+            this.#syncPreferredReadReplicas();
             callback(null);
         });
     }
@@ -1057,6 +1116,7 @@ export class Consumer extends Base {
                     return;
                 }
                 this.assignments = response;
+                this.#syncPreferredReadReplicas();
                 this.#cancelHeartbeat();
                 this.#heartbeatInterval = setTimeout(() => {
                     this.#heartbeat(options);
@@ -1126,6 +1186,7 @@ export class Consumer extends Base {
             this.memberId = null;
             this.generationId = 0;
             this.assignments = null;
+            this.#syncPreferredReadReplicas();
             callback(null);
         });
     }
@@ -1150,7 +1211,7 @@ export class Consumer extends Base {
                 }
                 this[kMetadata]({ topics: Array.from(topicsSubscriptions.keys()) }, (error, metadata) => {
                     if (error) {
-                        callback(this.#handleMetadataError(error));
+                        callback(this.#handleError(error));
                         return;
                     }
                     this.#performSyncGroup(partitionsAssigner, this.#createAssignments(partitionsAssigner, metadata), callback);
@@ -1188,26 +1249,30 @@ export class Consumer extends Base {
         }, callback);
     }
     #performGroupOperation(operationId, operation, callback) {
-        this.#findGroupCoordinator((error, coordinatorId) => {
-            if (error) {
-                callback(error);
-                return;
-            }
-            this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+        this[kPerformWithRetry](operationId, retryCallback => {
+            this.#findGroupCoordinator((error, coordinatorId) => {
                 if (error) {
-                    callback(this.#handleMetadataError(error));
+                    retryCallback(error);
                     return;
                 }
-                this[kPerformWithRetry](operationId, retryCallback => {
+                this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+                    if (error) {
+                        retryCallback(this.#handleError(error));
+                        return;
+                    }
                     this[kGetConnection](metadata.brokers.get(coordinatorId), (error, connection) => {
                         if (error) {
-                            retryCallback(error);
+                            retryCallback(this.#handleError(error));
                             return;
                         }
-                        operation(connection, retryCallback);
+                        operation(connection, (error, result) => {
+                            retryCallback(this.#handleError(error), result);
+                        });
                     });
-                }, callback);
+                });
             });
+        }, (error, result) => {
+            callback(this.#handleError(error), result);
         });
     }
     #validateGroupOptions(options, validator) {
@@ -1345,9 +1410,84 @@ export class Consumer extends Base {
         }
         return protocolError;
     }
-    #handleMetadataError(error) {
-        if (error && error?.findBy('hasStaleMetadata', true)) {
-            this.clearMetadata();
+    #updatePreferredReadReplicas(metadata, topicIds, response) {
+        for (const topicResponse of response.responses) {
+            const topic = topicIds.get(topicResponse.topicId);
+            if (!topic) {
+                continue;
+            }
+            for (const { partitionIndex: partition, preferredReadReplica } of topicResponse.partitions) {
+                if (preferredReadReplica < 0) {
+                    continue;
+                }
+                const partitionMetadata = metadata.topics.get(topic).partitions[partition];
+                const key = partitionKey(topic, partition);
+                if (metadata.brokers.has(preferredReadReplica) &&
+                    partitionMetadata.replicas.includes(preferredReadReplica) &&
+                    !partitionMetadata.offlineReplicas.includes(preferredReadReplica)) {
+                    const cachedPreferredReadReplica = this.#preferredReadReplicas.get(key);
+                    if (cachedPreferredReadReplica?.node !== preferredReadReplica) {
+                        this.#preferredReadReplicas.set(key, {
+                            node: preferredReadReplica,
+                            expiresAt: Date.now() + this[kOptions].metadataMaxAge
+                        });
+                    }
+                }
+                else {
+                    this.#preferredReadReplicas.delete(key);
+                    this.clearMetadata();
+                }
+            }
+        }
+    }
+    #clearPreferredReadReplicas(topics, topicIds) {
+        let cleared = false;
+        for (const topicRequest of topics) {
+            const topic = topicIds.get(topicRequest.topicId);
+            if (!topic) {
+                continue;
+            }
+            for (const { partition } of topicRequest.partitions) {
+                cleared = this.#preferredReadReplicas.delete(partitionKey(topic, partition)) || cleared;
+            }
+        }
+        return cleared;
+    }
+    // Drops cached preferred replicas for partitions that are no longer assigned.
+    // Called whenever `this.assignments` changes so the cache does not retain
+    // entries for partitions the consumer has lost during a rebalance.
+    #syncPreferredReadReplicas() {
+        if (this.#preferredReadReplicas.size === 0) {
+            return;
+        }
+        if (!this.assignments?.length) {
+            this.#preferredReadReplicas.clear();
+            return;
+        }
+        const assignedKeys = new Set();
+        for (const { topic, partitions } of this.assignments) {
+            for (const partition of partitions) {
+                assignedKeys.add(partitionKey(topic, partition));
+            }
+        }
+        for (const key of this.#preferredReadReplicas.keys()) {
+            if (!assignedKeys.has(key)) {
+                this.#preferredReadReplicas.delete(key);
+            }
+        }
+    }
+    #handleError(error) {
+        const kafkaError = error;
+        if (kafkaError) {
+            if (kafkaError.findBy('hasStaleMetadata', true)) {
+                this.clearMetadata();
+            }
+            else if (kafkaError.findBy('code', 'PLT_KFK_NETWORK') ||
+                kafkaError.findBy('apiId', 'COORDINATOR_LOAD_IN_PROGRESS') ||
+                kafkaError.findBy('apiId', 'COORDINATOR_NOT_AVAILABLE') ||
+                kafkaError.findBy('apiId', 'NOT_COORDINATOR')) {
+                this.#coordinatorId = null;
+            }
         }
         return error;
     }

package/dist/clients/consumer/messages-stream.js

@@ -5,11 +5,12 @@ import { consumerReceivesChannel, createDiagnosticContext, notifyCreation } from
 import { protocolErrors, UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
-import { kAutocommit, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
+import { kAutocommit, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { kConnections, kCreateConnectionPool, kInspect, kPrometheus } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { defaultConsumerOptions } from "./options.js";
 import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
+import { partitionKey } from "./utils.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
 /* c8 ignore next 3 - Fallback deserializer, nothing to really test */
@@ -145,7 +146,7 @@ export class MessagesStream extends Readable {
         this.#offsetsToFetch = new Map();
         if (offsets) {
             for (const { topic, partition, offset } of offsets) {
-                this.#offsetsToFetch.set(`${topic}:${partition}`, offset);
+                this.#offsetsToFetch.set(partitionKey(topic, partition), offset);
             }
         }
         // Clone the rest of the options so the user can never mutate them
@@ -364,21 +365,22 @@ export class MessagesStream extends Readable {
                 }
                 const partitions = assignment.partitions;
                 for (const partition of partitions) {
-                    const leader = metadata.topics.get(topic).partitions[partition].leader;
-                    if (this.#inflightNodes.has(leader)) {
+                    const targetNode = this.#consumer[kGetFetchNode](metadata, topic, partition, now);
+                    const key = partitionKey(topic, partition);
+                    if (this.#inflightNodes.has(targetNode)) {
                         continue;
                     }
-                    let leaderRequests = requests.get(leader);
-                    if (!leaderRequests) {
-                        leaderRequests = [];
-                        requests.set(leader, leaderRequests);
+                    let nodeRequests = requests.get(targetNode);
+                    if (!nodeRequests) {
+                        nodeRequests = [];
+                        requests.set(targetNode, nodeRequests);
                     }
                     const topicId = metadata.topics.get(topic).id;
                     topicIds.set(topicId, topic);
-                    const fetchOffset = this.#offsetsToFetch.get(`${topic}:${partition}`);
-                    requestedOffsets.set(`${topic}:${partition}`, fetchOffset);
-                    const leaderEpoch = this.#partitionsEpochs.get(`${topic}:${partition}`) ?? -1;
-                    leaderRequests.push({
+                    const fetchOffset = this.#offsetsToFetch.get(key);
+                    requestedOffsets.set(key, fetchOffset);
+                    const leaderEpoch = this.#partitionsEpochs.get(key) ?? -1;
+                    nodeRequests.push({
                         topicId,
                         partitions: [
                             {
@@ -395,10 +397,10 @@ export class MessagesStream extends Readable {
             if (requests.size === 0) {
                 return;
             }
-            for (const [leader, leaderRequests] of requests) {
-                this.#inflightNodes.set(leader, Date.now());
-                this.#consumer.fetch({ ...this.#options, node: leader, topics: leaderRequests, connectionPool: this[kConnections] }, (error, response) => {
-                    this.#inflightNodes.delete(leader);
+            for (const [node, nodeRequests] of requests) {
+                this.#inflightNodes.set(node, Date.now());
+                this.#consumer.fetch({ ...this.#options, node, topics: nodeRequests, connectionPool: this[kConnections] }, (error, response) => {
+                    this.#inflightNodes.delete(node);
                     this.emit('fetch');
                     if (error) {
                         // The stream has been closed, ignore the error
@@ -407,10 +409,22 @@ export class MessagesStream extends Readable {
                             this.push(null);
                             return;
                         }
-                        if (this.#fallbackMode !== MessagesStreamFallbackModes.FAIL &&
-                            this.#handleOffsetOutOfRange(error, topicIds)) {
-                            process.nextTick(() => {
-                                this.#fetch();
+                        if (this.#fallbackMode !== MessagesStreamFallbackModes.FAIL) {
+                            this.#handleOffsetOutOfRange(error, topicIds, (recoveryError, recovered) => {
+                                if (this.#closed || this.closed || this.destroyed) {
+                                    return;
+                                }
+                                if (recoveryError) {
+                                    this.destroy(recoveryError);
+                                    return;
+                                }
+                                if (recovered) {
+                                    process.nextTick(() => {
+                                        this.#fetch();
+                                    });
+                                    return;
+                                }
+                                this.destroy(error);
                             });
                             return;
                         }
@@ -430,34 +444,81 @@ export class MessagesStream extends Readable {
             }
         });
     }
-    #handleOffsetOutOfRange(error, topicIds) {
+    #handleOffsetOutOfRange(error, topicIds, callback) {
         if (!error.findBy?.('apiId', 'OFFSET_OUT_OF_RANGE')) {
-            return false;
+            callback(null, false);
+            return;
         }
         const response = error.response;
         if (!response || response.errorCode !== 0) {
-            return false;
+            callback(null, false);
+            return;
         }
         const recoveredOffsets = [];
+        const partitionsToRefresh = new Map();
         for (const topicResponse of response.responses) {
             const topic = topicIds.get(topicResponse.topicId);
             if (!topic) {
-                return false;
+                callback(null, false);
+                return;
             }
             for (const partitionResponse of topicResponse.partitions) {
                 if (partitionResponse.errorCode === 0) {
                     continue;
                 }
                 if (partitionResponse.errorCode !== protocolErrors.OFFSET_OUT_OF_RANGE.code) {
-                    return false;
+                    callback(null, false);
+                    return;
                 }
-                const key = `${topic}:${partitionResponse.partitionIndex}`;
                 const offset = this.#fallbackMode === MessagesStreamFallbackModes.EARLIEST
                     ? partitionResponse.logStartOffset
                     : partitionResponse.highWatermark;
-                recoveredOffsets.push([key, offset]);
+                if (offset >= 0n) {
+                    recoveredOffsets.push([partitionKey(topic, partitionResponse.partitionIndex), offset]);
+                    continue;
+                }
+                let partitions = partitionsToRefresh.get(topic);
+                if (!partitions) {
+                    partitions = [];
+                    partitionsToRefresh.set(topic, partitions);
+                }
+                partitions.push(partitionResponse.partitionIndex);
             }
         }
+        if (partitionsToRefresh.size > 0) {
+            this.#refreshOutOfRangeOffsets(partitionsToRefresh, recoveredOffsets, callback);
+            return;
+        }
+        callback(null, this.#applyRecoveredOffsets(recoveredOffsets));
+    }
+    #refreshOutOfRangeOffsets(partitionsToRefresh, recoveredOffsets, callback) {
+        const partitions = Object.fromEntries(partitionsToRefresh);
+        this.#consumer.listOffsets({
+            topics: Array.from(partitionsToRefresh.keys()),
+            partitions,
+            timestamp: this.#fallbackMode === MessagesStreamFallbackModes.EARLIEST
+                ? ListOffsetTimestamps.EARLIEST
+                : ListOffsetTimestamps.LATEST
+        }, (error, offsets) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            for (const [topic, refreshedPartitions] of partitionsToRefresh) {
+                const topicOffsets = offsets.get(topic);
+                for (const partition of refreshedPartitions) {
+                    const offset = topicOffsets?.[partition];
+                    if (typeof offset !== 'bigint' || offset < 0n) {
+                        callback(new UserError(`Cannot recover offset out of range for topic ${topic} partition ${partition}.`));
+                        return;
+                    }
+                    recoveredOffsets.push([partitionKey(topic, partition), offset]);
+                }
+            }
+            callback(null, this.#applyRecoveredOffsets(recoveredOffsets));
+        });
+    }
+    #applyRecoveredOffsets(recoveredOffsets) {
         for (const [key, offset] of recoveredOffsets) {
             this.#offsetsToFetch.set(key, offset);
             this.#offsetsCommitted.set(key, offset);
@@ -487,6 +548,7 @@ export class MessagesStream extends Readable {
         for (const topicResponse of response.responses) {
             const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                const key = partitionKey(topic, partition);
                 if (!recordsBatches) {
                     continue;
                 }
@@ -494,15 +556,15 @@ export class MessagesStream extends Readable {
                     const firstTimestamp = batch.firstTimestamp;
                     const firstOffset = batch.firstOffset;
                     const leaderEpoch = metadata.topics.get(topic).partitions[partition].leaderEpoch;
-                    this.#partitionsEpochs.set(`${topic}:${partition}`, leaderEpoch);
+                    this.#partitionsEpochs.set(key, leaderEpoch);
                     // Track offsets
                     if (batch === recordsBatches[recordsBatches.length - 1]) {
                         // Track the last read offset
                         const lastOffset = batch.firstOffset + BigInt(batch.lastOffsetDelta);
-                        this.#offsetsToFetch.set(`${topic}:${partition}`, lastOffset + 1n);
+                        this.#offsetsToFetch.set(key, lastOffset + 1n);
                         // Autocommit if needed
                         if (autocommit) {
-                            this.#offsetsToCommit.set(`${topic}:${partition}`, {
+                            this.#offsetsToCommit.set(key, {
                                 topic,
                                 partition,
                                 offset: lastOffset + 1n,
@@ -518,7 +580,7 @@ export class MessagesStream extends Readable {
                     for (const record of batch.records) {
                         const messageToConsume = { ...record, topic, partition };
                         const offset = batch.firstOffset + BigInt(record.offsetDelta);
-                        if (offset < requestedOffsets.get(`${topic}:${partition}`)) {
+                        if (offset < requestedOffsets.get(key)) {
                             // Thi is a duplicate message, ignore it
                             continue;
                         }
@@ -605,7 +667,7 @@ export class MessagesStream extends Readable {
         }
     }
     #updateCommittedOffset(topic, partition, offset) {
-        const key = `${topic}:${partition}`;
+        const key = partitionKey(topic, partition);
         const previous = this.#offsetsCommitted.get(key);
         if (typeof previous === 'undefined' || previous < offset) {
             this.#offsetsCommitted.set(key, offset);
@@ -767,8 +829,9 @@ export class MessagesStream extends Readable {
             }
             const partitions = assignment.partitions;
             for (const partition of partitions) {
-                const committed = this.#offsetsToFetch.get(`${topic}:${partition}`);
-                this.#offsetsCommitted.set(`${topic}:${partition}`, committed);
+                const key = partitionKey(topic, partition);
+                const committed = this.#offsetsToFetch.get(key);
+                this.#offsetsCommitted.set(key, committed);
             }
         }
         this.emit('offsets');

package/dist/clients/consumer/types.d.ts

@@ -27,6 +27,10 @@ export type OffsetsWithTimestamps = Map<string, Map<number, {
     offset: bigint;
     timestamp: bigint;
 }>>;
+export interface PreferredReadReplica {
+    node: number;
+    expiresAt: number;
+}
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
 export declare const MessagesStreamModes: {

package/dist/clients/consumer/utils.d.ts

@@ -0,0 +1 @@
+export declare function partitionKey(topic: string, partition: number): string;

package/dist/clients/consumer/utils.js

@@ -0,0 +1,3 @@
+export function partitionKey(topic, partition) {
+    return `${topic}:${partition}`;
+}

package/dist/clients/producer/producer.d.ts

@@ -13,7 +13,7 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     get producerId(): bigint | undefined;
     get producerEpoch(): number | undefined;
     get transaction(): Transaction<Key, Value, HeaderKey, HeaderValue> | undefined;
-    get coordinatorId(): number;
+    get coordinatorId(): number | undefined;
     get streamsCount(): number;
     close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     close(force?: boolean): Promise<void>;

package/dist/clients/producer/producer.js

@@ -383,12 +383,12 @@ export class Producer extends Base {
             this[kPerformWithRetry]('addPartitionsToTransaction', retryCallback => {
                 this.#getCoordinatorConnection((error, connection) => {
                     if (error) {
-                        retryCallback(error);
+                        retryCallback(this.#handleError(error));
                         return;
                     }
                     this[kGetApi]('AddPartitionsToTxn', (error, api) => {
                         if (error) {
-                            retryCallback(error);
+                            retryCallback(this.#handleError(error));
                             return;
                         }
                         api(connection, [
@@ -401,7 +401,7 @@ export class Producer extends Base {
                             }
                         ], error => {
                             if (error) {
-                                retryCallback(error);
+                                retryCallback(this.#handleError(error));
                                 return;
                             }
                             retryCallback(null);
@@ -420,17 +420,17 @@ export class Producer extends Base {
             this[kPerformWithRetry]('addOffsetsToTransaction', retryCallback => {
                 this.#getCoordinatorConnection((error, connection) => {
                     if (error) {
-                        retryCallback(error);
+                        retryCallback(this.#handleError(error));
                         return;
                     }
                     this[kGetApi]('AddOffsetsToTxn', (error, api) => {
                         if (error) {
-                            retryCallback(error);
+                            retryCallback(this.#handleError(error));
                             return;
                         }
                         api(connection, this.#transaction.id, this.#producerInfo.producerId, this.#producerInfo.producerEpoch, groupId, error => {
                             if (error) {
-                                retryCallback(error);
+                                retryCallback(this.#handleError(error));
                                 return;
                             }
                             retryCallback(null);
@@ -498,18 +498,18 @@ export class Producer extends Base {
             this[kPerformWithRetry]('endTransaction', retryCallback => {
                 this.#getCoordinatorConnection((error, connection) => {
                     if (error) {
-                        retryCallback(error);
+                        retryCallback(this.#handleError(error));
                         return;
                     }
                     this[kGetApi]('EndTxn', (error, api) => {
                         if (error) {
-                            retryCallback(error);
+                            retryCallback(this.#handleError(error));
                             return;
                         }
                         api(connection, this.#transaction.id, this.#producerInfo.producerId, this.#producerInfo.producerEpoch, commit, error => {
                             if (error) {
                                 this.#handleFencingError(error);
-                                retryCallback(error);
+                                retryCallback(this.#handleError(error));
                                 return;
                             }
                             this.#transaction = undefined;
@@ -545,13 +545,15 @@ export class Producer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, transactionalId, this[kOptions].timeout, options.producerId ?? this[kOptions].producerId ?? 0n, options.producerEpoch ?? this[kOptions].producerEpoch ?? 0, retryCallback);
+                        api(connection, transactionalId, this[kOptions].timeout, options.producerId ?? this[kOptions].producerId ?? 0n, options.producerEpoch ?? this[kOptions].producerEpoch ?? 0, (error, response) => {
+                            retryCallback(this.#handleError(error), response);
+                        });
                     });
                 });
             }, (error, response) => {
                 if (error) {
                     this.#handleFencingError(error);
-                    deduplicateCallback(error);
+                    deduplicateCallback(this.#handleError(error));
                     return;
                 }
                 this.#producerInfo = {
@@ -777,21 +779,29 @@ export class Producer extends Base {
         });
     }
     #getCoordinatorConnection(callback) {
+        if (this.#coordinatorId === undefined) {
+            this[kTransactionFindCoordinator](error => {
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                this.#getCoordinatorConnection(callback);
+            });
+            return;
+        }
         // Get a connection to the coordinator
         this[kMetadata]({}, (error, metadata) => {
             if (error) {
                 callback(error);
                 return;
             }
-            this[kPerformWithRetry]('getCoordinatorConnection', retryCallback => {
-                this[kGetConnection](metadata.brokers.get(this.#coordinatorId), (error, connection) => {
-                    if (error) {
-                        retryCallback(error);
-                        return;
-                    }
-                    retryCallback(null, connection);
-                });
-            }, callback);
+            this[kGetConnection](metadata.brokers.get(this.#coordinatorId), (error, connection) => {
+                if (error) {
+                    callback(this.#handleError(error));
+                    return;
+                }
+                callback(null, connection);
+            });
         });
     }
     #handleFencingError(error) {
@@ -800,6 +810,18 @@ export class Producer extends Base {
             this.#transaction = undefined;
         }
     }
+    #handleError(error) {
+        const kafkaError = error;
+        if (kafkaError) {
+            if (kafkaError.findBy('code', 'PLT_KFK_NETWORK') ||
+                kafkaError.findBy('apiId', 'COORDINATOR_LOAD_IN_PROGRESS') ||
+                kafkaError.findBy('apiId', 'COORDINATOR_NOT_AVAILABLE') ||
+                kafkaError.findBy('apiId', 'NOT_COORDINATOR')) {
+                this.#coordinatorId = undefined;
+            }
+        }
+        return error;
+    }
     #beforeSerialization(hook, options, callback) {
         // Create the pre-serialization requests
         const requests = [];

package/dist/network/connection.js

@@ -55,7 +55,12 @@ export class Connection extends TypedEventEmitter {
         super();
         this.setMaxListeners(0);
         this.#instanceId = currentInstance++;
-        this.#options = Object.assign({}, defaultOptions, options);
+        this.#options = Object.assign({}, defaultOptions);
+        for (const [key, value] of Object.entries(options)) {
+            if (value !== undefined) {
+                this.#options[key] = value;
+            }
+        }
         this.#options.tls ??= this.#options.ssl;
         this.#status = ConnectionStatuses.NONE;
         this.#clientId = clientId;
@@ -124,7 +129,7 @@ export class Connection extends TypedEventEmitter {
             }
             /* c8 ignore next 13 - Hard to test */
             const connectingSocketTimeoutHandler = () => {
-                const error = new TimeoutError(`Connection to ${host}:${port} timed out.`);
+                const error = new TimeoutError(`Connection to ${host}:${port} timed out.`, { canRetry: true });
                 diagnosticContext.error = error;
                 this.#socket.destroy();
                 this.#status = ConnectionStatuses.ERROR;
@@ -214,7 +219,7 @@ export class Connection extends TypedEventEmitter {
             this.#socket?.destroy();
             callback(new TimeoutError(this.#host
                 ? `Connection to ${this.#host}:${this.#port} timed out.`
-                : `Connection ready timed out after ${this.#options.connectTimeout}ms.`));
+                : `Connection ready timed out after ${this.#options.connectTimeout}ms.`, { canRetry: true }));
         }, this.#options.connectTimeout);
         this.once('connect', onConnect);
         this.once('error', onError);
@@ -253,7 +258,9 @@ export class Connection extends TypedEventEmitter {
         return callback[kCallbackPromise];
     }
     send(apiKey, apiVersion, createPayload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
-        const correlationId = ++this.#correlationId;
+        // Correlation ID is a 32-bit integer in the protocol, so we need to wrap around after 2^31 - 1
+        const correlationId = (this.#correlationId + 1) & 0x7FFFFFFF;
+        this.#correlationId = correlationId;
         const diagnostic = createDiagnosticContext({
             connection: this,
             operation: 'send',

package/dist/registries/confluent-schema-registry.js

@@ -60,8 +60,9 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
             if (!response.ok) {
                 throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, { response: await response.text() });
             }
-            const responseBody = await response.json();
-            const { schema, schemaType } = responseBody;
+            const responseBody = (await response.json());
+            const { schema } = responseBody;
+            const schemaType = responseBody.schemaType ?? 'AVRO';
             switch (schemaType) {
                 case 'AVRO':
                     this.#schemas.set(id, { id, type: 'avro', schema: avro.Type.forSchema(JSON.parse(schema)) });

package/dist/symbols.d.ts

@@ -1,6 +1,7 @@
 export declare const kInstance: unique symbol;
 export declare const kRefreshOffsetsAndFetch: unique symbol;
 export declare const kAutocommit: unique symbol;
+export declare const kGetFetchNode: unique symbol;
 export declare const kTransaction: unique symbol;
 export declare const kTransactionPrepare: unique symbol;
 export declare const kTransactionAddPartitions: unique symbol;

package/dist/symbols.js

@@ -2,6 +2,7 @@
 export const kInstance = Symbol('plt.kafka.base.instance');
 export const kRefreshOffsetsAndFetch = Symbol('plt.kafka.messagesStream.refreshOffsetsAndFetch');
 export const kAutocommit = Symbol('plt.kafka.messagesStream.autocommit');
+export const kGetFetchNode = Symbol('plt.kafka.consumer.getFetchNode');
 export const kTransaction = Symbol('plt.kafka.producer.transactions');
 export const kTransactionPrepare = Symbol('plt.kafka.producer.transactions.prepare');
 export const kTransactionAddPartitions = Symbol('plt.kafka.producer.transactions.addPartitions');

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -84,11 +84,11 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
   [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
   clearMetadata(): void;
-  [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+  [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
   [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
   [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
   [kGetConnection](broker: Broker, callback: Callback<Connection>): void;
-  [kGetBootstrapConnection](callback: Callback<Connection>): void;
+  [kGetBootstrapConnection](callback: Callback<Connection>, attempt?: number): void;
   [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
   [kInspect](...args: unknown[]): void;
   [kFormatValidationErrors](validator: ValidateFunction<unknown>, targetName: string): string;

package/dist/typescript-4/dist/clients/base/options.d.ts

@@ -25,6 +25,9 @@ export declare const baseOptionsSchema: {
       type: string;
       pattern: string;
     };
+    clientRack: {
+      type: string;
+    };
     bootstrapBrokers: {
       oneOf: ({
         type: string;

package/dist/typescript-4/dist/clients/base/types.d.ts

@@ -32,6 +32,7 @@ export interface ClusterMetadata {
 export type RetryDelayGetter<Owner = object> = (client: Owner, operationId: string, attempt: number, retries: number, error: Error) => number;
 export interface BaseOptions extends ConnectionOptions {
   clientId: string;
+  clientRack?: string;
   bootstrapBrokers: Broker[] | string[];
   context?: unknown;
   timeout?: number;

package/dist/typescript-4/dist/clients/consumer/consumer.d.ts

@@ -1,7 +1,9 @@
 import { type CallbackWithPromise } from "../../apis/callbacks";
 import { type FetchResponse } from "../../apis/consumer/fetch-v17";
 import { type ConnectionPool } from "../../network/connection-pool";
+import { kGetFetchNode } from "../../symbols";
 import { Base, type BaseEvents, kCreateConnectionPool } from "../base/base";
+import { type ClusterMetadata } from "../base/types";
 import { MessagesStream } from "./messages-stream";
 import { TopicsMap } from "./topics-map";
 import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
@@ -56,4 +58,5 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   joinGroup(options?: GroupOptions): Promise<string>;
   leaveGroup(force?: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
   leaveGroup(force?: boolean): Promise<void>;
+  [kGetFetchNode](metadata: ClusterMetadata, topic: string, partition: number, now: number): number;
 }

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -27,6 +27,10 @@ export type OffsetsWithTimestamps = Map<string, Map<number, {
   offset: bigint;
   timestamp: bigint;
 }>>;
+export interface PreferredReadReplica {
+  node: number;
+  expiresAt: number;
+}
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
 export declare const MessagesStreamModes: {

package/dist/typescript-4/dist/clients/consumer/utils.d.ts

@@ -0,0 +1 @@
+export declare function partitionKey(topic: string, partition: number): string;

package/dist/typescript-4/dist/clients/producer/producer.d.ts

@@ -13,7 +13,7 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   get producerId(): bigint | undefined;
   get producerEpoch(): number | undefined;
   get transaction(): Transaction<Key, Value, HeaderKey, HeaderValue> | undefined;
-  get coordinatorId(): number;
+  get coordinatorId(): number | undefined;
   get streamsCount(): number;
   close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
   close(force?: boolean): Promise<void>;

package/dist/typescript-4/dist/symbols.d.ts

@@ -1,6 +1,7 @@
 export declare const kInstance: unique symbol;
 export declare const kRefreshOffsetsAndFetch: unique symbol;
 export declare const kAutocommit: unique symbol;
+export declare const kGetFetchNode: unique symbol;
 export declare const kTransaction: unique symbol;
 export declare const kTransactionPrepare: unique symbol;
 export declare const kTransactionAddPartitions: unique symbol;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.0.1";
+export const version = "2.2.2";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.0.1",
+  "version": "2.2.2",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -89,12 +89,19 @@
     "postbuild": "./scripts/node scripts/postbuild.ts",
     "lint": "eslint --cache",
     "typecheck": "tsc -p . --noEmit",
-    "format": "prettier -w benchmarks playground src test",
+    "format": "prettier -w benchmarks playground regression src test",
     "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
     "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:integrity": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/integrity/*.test.ts'",
+    "test:performance": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/performance/*.test.ts'",
+    "test:regression": "npm run test:integrity",
+    "test:regression:all": "npm run test:integrity && npm run test:performance",
+    "test:all": "npm test && npm run test:regression:all",
     "test:memory": "NODE_OPTIONS='--expose-gc' ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/**/*.memory-test.ts'",
     "test:docker:up": "docker-compose up -d --wait",
     "test:docker:down": "docker-compose down",
+    "regression:baseline:compare": "npm run test:performance",
+    "regression:baseline:update": "./scripts/node regression/helpers/run-benchmarks.ts",
     "ci": "npm run build && npm run lint && npm run test:ci",
     "generate:apis": "node scripts/generate-apis.ts",
     "generate:errors": "node scripts/generate-errors.ts",