@platformatic/kafka 1.33.2 → 1.35.0

package/dist/apis/admin/alter-partition-reassignments-v1.js

@@ -39,6 +39,7 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
     const throttleTimeMs = reader.readInt32();
     const errorCode = reader.readInt16();
     const errorMessage = reader.readNullableString();
+    /* c8 ignore next 3 - Hard to test */
     if (errorCode !== 0) {
         errors.push(['', [errorCode, errorMessage]]);
     }

package/dist/clients/admin/admin.d.ts

@@ -5,7 +5,7 @@ import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type Callback } from '../../apis/definitions.ts';
 import { type Acl } from '../../apis/types.ts';
 import { Base } from '../base/base.ts';
-import { type AdminListOffsetsOptions, type AdminOptions, type AlterClientQuotasOptions, type AlterConfigsOptions, type AlterConsumerGroupOffsetsOptions, type BrokerLogDirDescription, type ConfigDescription, type CreateAclsOptions, type CreatedTopic, type CreatePartitionsOptions, type CreateTopicsOptions, type DeleteAclsOptions, type DeleteConsumerGroupOffsetsOptions, type DeleteRecordsOptions, type DeleteGroupsOptions, type DeleteTopicsOptions, type DeletedRecordsTopic, type DescribeAclsOptions, type DescribeClientQuotasOptions, type DescribeConfigsOptions, type DescribeGroupsOptions, type DescribeLogDirsOptions, type FindCoordinatorOptions, type FindCoordinatorResult, type Group, type GroupBase, type IncrementalAlterConfigsOptions, type ListConsumerGroupOffsetsGroup, type ListConsumerGroupOffsetsOptions, type ListedOffsetsTopic, type ListGroupsOptions, type ListTopicsOptions, type RemoveMembersFromConsumerGroupOptions } from './types.ts';
+import { type AdminListOffsetsOptions, type AdminOptions, type AlterClientQuotasOptions, type AlterConfigsOptions, type AlterConsumerGroupOffsetsOptions, type BrokerLogDirDescription, type ConfigDescription, type CreateAclsOptions, type CreatedTopic, type CreatePartitionsOptions, type CreateTopicsOptions, type DeleteAclsOptions, type DeleteConsumerGroupOffsetsOptions, type DeletedRecordsTopic, type DeleteGroupsOptions, type DeleteRecordsOptions, type DeleteTopicsOptions, type DescribeAclsOptions, type DescribeClientQuotasOptions, type DescribeConfigsOptions, type DescribeGroupsOptions, type DescribeLogDirsOptions, type FindCoordinatorOptions, type FindCoordinatorResult, type Group, type GroupBase, type IncrementalAlterConfigsOptions, type ListConsumerGroupOffsetsGroup, type ListConsumerGroupOffsetsOptions, type ListedOffsetsTopic, type ListGroupsOptions, type ListTopicsOptions, type RemoveMembersFromConsumerGroupOptions } from './types.ts';
 export declare class Admin extends Base<AdminOptions> {
     #private;
     constructor(options: AdminOptions);

package/dist/clients/admin/admin.js

@@ -4,7 +4,7 @@ import { adminAclsChannel, adminClientQuotasChannel, adminConfigsChannel, adminC
 import { MultipleErrors, UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Base, kAfterCreate, kCheckNotClosed, kConnections, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
-import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteRecordsOptionsValidator, deleteGroupsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, findCoordinatorOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
+import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteGroupsOptionsValidator, deleteRecordsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, findCoordinatorOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
 export class Admin extends Base {
     #controller = null;
     constructor(options) {
@@ -1391,6 +1391,7 @@ export class Admin extends Base {
                     // topics.get(topic.name) must be defined as the metadata request was successful
                     const topicData = topics.get(topic.name);
                     const targetPartitionData = topicData.partitions[partition.partitionIndex];
+                    /* c8 ignore next 4 - Hard to test */
                     if (!targetPartitionData) {
                         callback(new UserError(`Unknown partition ${partition.partitionIndex} for topic ${topic.name}.`));
                         return;

package/dist/clients/base/base.d.ts

@@ -66,6 +66,8 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     get closed(): boolean;
     get type(): ClientType;
     get context(): unknown;
+    get currentMetadata(): ClusterMetadata | undefined;
+    get connections(): ConnectionPool;
     emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;

package/dist/clients/base/base.js

@@ -90,6 +90,12 @@ export class Base extends TypedEventEmitter {
     get context() {
         return this[kContext];
     }
+    get currentMetadata() {
+        return this.#metadata;
+    }
+    get connections() {
+        return this[kConnections];
+    }
     emitWithDebug(section, name, ...args) {
         if (!section) {
             return this.emit(name, ...args);
@@ -254,14 +260,19 @@ export class Base extends TypedEventEmitter {
                     this.once('client:close', onClose);
                 }
                 else {
-                    if (attempt === 0) {
-                        callback(error);
-                        return;
+                    if (attempt > 0) {
+                        error = new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors);
                     }
-                    callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors));
+                    this.emitWithDebug('client', 'performWithRetry:failed', operationId, retries, errors);
+                    callback(error);
+                    return;
                 }
                 return;
             }
+            // Make sure all previous errors are deleted
+            if (errors.length > 0) {
+                errors.splice(0, errors.length);
+            }
             callback(null, result);
         });
         return callback[kCallbackPromise];
@@ -398,6 +409,7 @@ export class Base extends TypedEventEmitter {
                     const unknownTopicError = error.findBy('apiCode', 3);
                     if (unknownTopicError) {
                         const topicIndexMatch = unknownTopicError.path?.match(/\/topics\/(\d+)/);
+                        /* c8 ignore next 3 - Hard to test  */
                         const topicIndex = topicIndexMatch ? parseInt(topicIndexMatch[1]) : -1;
                         const topicName = topicIndex >= 0 && topicIndex < topicsToFetch.length ? topicsToFetch[topicIndex] : 'unknown';
                         deduplicateCallback(new UserError(`Unknown topic ${topicName}.`));
@@ -405,6 +417,7 @@ export class Base extends TypedEventEmitter {
                     }
                     const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
                     // Stale metadata, we need to fetch everything again
+                    /* c8 ignore next 4 - Hard to test */
                     if (hasStaleMetadata) {
                         this.clearMetadata();
                         topicsToFetch = topics;

package/dist/clients/consumer/consumer.js

@@ -172,6 +172,7 @@ export class Consumer extends Base {
         }
         options.autocommit ??= this[kOptions].autocommit ?? true;
         options.maxBytes ??= this[kOptions].maxBytes;
+        options.maxBytesPerPartition ??= this[kOptions].maxBytesPerPartition ?? options.maxBytes;
         options.highWaterMark ??= this[kOptions].highWaterMark;
         options.registry ??= this[kOptions].registry;
         options.beforeDeserialization ??= this[kOptions].beforeDeserialization;
@@ -483,7 +484,7 @@ export class Consumer extends Base {
             callback(error, response);
         }, 0);
     }
-    #commit(options, callback) {
+    #commit(options, callback, rejoinAttempts = 0) {
         this.#performGroupOperation('commit', (connection, groupCallback) => {
             const topics = new Map();
             for (const { topic, partition, offset, leaderEpoch } of options.offsets) {
@@ -507,6 +508,16 @@ export class Consumer extends Base {
                 api(connection, this.groupId, this.#useConsumerGroupProtocol ? this.#memberEpoch : this.generationId, this.memberId, null, Array.from(topics.values()), groupCallback);
             });
         }, error => {
+            if (error && rejoinAttempts < this[kOptions].retries && this.#getRejoinError(error)) {
+                this.joinGroup({}, joinError => {
+                    if (joinError) {
+                        callback(joinError);
+                        return;
+                    }
+                    this.#commit(options, callback, rejoinAttempts + 1);
+                });
+                return;
+            }
             callback(error);
         });
     }
@@ -608,7 +619,7 @@ export class Consumer extends Base {
             });
         });
     }
-    #listCommittedOffsets(options, callback) {
+    #listCommittedOffsets(options, callback, staleMemberEpochRetries = 0) {
         const topics = [];
         for (const { topic: name, partitions } of options.topics) {
             topics.push({ name, partitionIndexes: partitions });
@@ -630,6 +641,18 @@ export class Consumer extends Base {
             });
         }, (error, response) => {
             if (error) {
+                if (this.#useConsumerGroupProtocol &&
+                    staleMemberEpochRetries < this[kOptions].retries &&
+                    error.findBy?.('apiId', 'STALE_MEMBER_EPOCH')) {
+                    this.#consumerGroupHeartbeat(this[kOptions], heartbeatError => {
+                        if (heartbeatError) {
+                            callback(this.#handleMetadataError(heartbeatError));
+                            return;
+                        }
+                        this.#listCommittedOffsets(options, callback, staleMemberEpochRetries + 1);
+                    });
+                    return;
+                }
                 callback(this.#handleMetadataError(error));
                 return;
             }

package/dist/clients/consumer/messages-stream.d.ts

@@ -19,6 +19,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     get offsetsToCommit(): Map<string, CommitOptionsPartition>;
     get offsetsCommitted(): Map<string, bigint>;
     get committedOffsets(): Map<string, bigint>;
+    get connections(): ConnectionPool;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
     isActive(): boolean;

package/dist/clients/consumer/messages-stream.js

@@ -61,6 +61,8 @@ export class MessagesStream extends Readable {
     #metricsConsumedMessages;
     #corruptedMessageHandler;
     #context;
+    #onConsumerGroupJoin;
+    #onBrokerDisconnect;
     #pushRecordsOperation;
     [kInstance];
     /*
@@ -122,6 +124,14 @@ export class MessagesStream extends Readable {
         this.#closeCallbacks = [];
         this.#corruptedMessageHandler = onCorruptedMessage ?? defaultCorruptedMessageHandler;
         this.#context = context;
+        this.#onConsumerGroupJoin = () => {
+            this.#offsetsCommitted.clear();
+            this.#partitionsEpochs.clear();
+            this.#scheduleRefreshOffsetsAndFetch();
+        };
+        this.#onBrokerDisconnect = () => {
+            this.#partitionsEpochs.clear();
+        };
         if (registry) {
             this.#pushRecordsOperation = this.#beforeDeserialization.bind(this, registry.getBeforeDeserializationHook());
         }
@@ -150,18 +160,12 @@ export class MessagesStream extends Readable {
         // When the consumer joins a group, we need to fetch again as the assignments
         // will have changed so we may have gone from last  with no assignments to
         // having some.
-        this.#consumer.on('consumer:group:join', () => {
-            this.#offsetsCommitted.clear();
-            this.#partitionsEpochs.clear();
-            this.#scheduleRefreshOffsetsAndFetch();
-        });
+        this.#consumer.on('consumer:group:join', this.#onConsumerGroupJoin);
         if (consumer[kPrometheus]) {
             this.#metricsConsumedMessages = ensureMetric(consumer[kPrometheus], 'Counter', 'kafka_consumed_messages', 'Number of consumed Kafka messages');
         }
         // Whenever the consumer loses a connection, reset all the partitions epochs
-        consumer.on('client:broker:disconnect', () => {
-            this.#partitionsEpochs.clear();
-        });
+        consumer.on('client:broker:disconnect', this.#onBrokerDisconnect);
         notifyCreation('messages-stream', this);
     }
     /* c8 ignore next 3 - Simple getter */
@@ -188,6 +192,10 @@ export class MessagesStream extends Readable {
     get committedOffsets() {
         return this.#offsetsCommitted;
     }
+    /* c8 ignore next 3 - Simple getter */
+    get connections() {
+        return this[kConnections];
+    }
     close(callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -298,12 +306,23 @@ export class MessagesStream extends Readable {
         return super[Symbol.asyncIterator]();
     }
     _construct(callback) {
-        this.#refreshOffsets(callback);
+        this.#refreshOffsetsInflight = true;
+        this.#refreshOffsets(error => {
+            const wasPending = this.#refreshOffsetsPending;
+            this.#refreshOffsetsInflight = false;
+            this.#refreshOffsetsPending = false;
+            if (!error && wasPending) {
+                this.#scheduleRefreshOffsetsAndFetch();
+            }
+            callback(error ?? undefined);
+        });
     }
     _destroy(error, callback) {
         if (this.#autocommitInterval) {
             clearInterval(this.#autocommitInterval);
         }
+        this.#consumer.removeListener('consumer:group:join', this.#onConsumerGroupJoin);
+        this.#consumer.removeListener('client:broker:disconnect', this.#onBrokerDisconnect);
         this[kConnections].close(closeError => {
             callback(closeError ?? error);
         });
@@ -379,7 +398,7 @@ export class MessagesStream extends Readable {
                             {
                                 partition,
                                 fetchOffset,
-                                partitionMaxBytes: this.#options.maxBytes,
+                                partitionMaxBytes: this.#options.maxBytesPerPartition,
                                 currentLeaderEpoch: leaderEpoch,
                                 lastFetchedEpoch: leaderEpoch
                             }
@@ -589,7 +608,12 @@ export class MessagesStream extends Readable {
             this.#autocommitInflight = false;
             if (error) {
                 this.emit('autocommit', error);
-                this.destroy(error);
+                // Only destroy the stream when the broker requires a group rejoin
+                // (ILLEGAL_GENERATION, UNKNOWN_MEMBER_ID, REBALANCE_IN_PROGRESS).
+                // Transient coordinator errors must not tear down the consumption loop.
+                if (error.findBy?.('needsRejoin', true)) {
+                    this.destroy(error);
+                }
                 return;
             }
             for (const { topic, partition, offset } of offsets) {
@@ -635,7 +659,10 @@ export class MessagesStream extends Readable {
                 topics.push(assignment);
             }
             if (!topics.length) {
-                this.#assignOffsets(offsets, new Map(), callback);
+                // No assignment yet: do not seed fallback offsets for the whole topic.
+                // consumer:group:join will refresh once partitions are assigned.
+                this.emit('offsets');
+                callback(null);
                 return;
             }
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
@@ -662,6 +689,7 @@ export class MessagesStream extends Readable {
         this.#refreshOffsetsInflight = true;
         this.#refreshOffsets(error => {
             const shouldDestroyOnError = this.#refreshOffsetsDestroyOnError;
+            const wasPending = this.#refreshOffsetsPending;
             this.#refreshOffsetsInflight = false;
             this.#refreshOffsetsDestroyOnError = false;
             this.#refreshOffsetsPending = false;
@@ -673,8 +701,7 @@ export class MessagesStream extends Readable {
                 return;
             }
             // A new one was scheduled while the previous one was inflight, we need to run it immediately
-            /* c8 ignore next 4 - Hard to test */
-            if (this.#refreshOffsetsPending) {
+            if (wasPending) {
                 this.#scheduleRefreshOffsetsAndFetch(shouldDestroyOnError);
                 return;
             }

package/dist/clients/consumer/options.d.ts

@@ -97,6 +97,10 @@ export declare const consumeOptionsProperties: {
         type: string;
         minimum: number;
     };
+    maxBytesPerPartition: {
+        type: string;
+        minimum: number;
+    };
     maxWaitTime: {
         type: string;
         minimum: number;
@@ -217,6 +221,10 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;
@@ -379,6 +387,10 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;
@@ -502,6 +514,10 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;

package/dist/clients/consumer/options.js

@@ -58,6 +58,7 @@ export const consumeOptionsProperties = {
     autocommit: { oneOf: [{ type: 'boolean' }, { type: 'number', minimum: 100 }] },
     minBytes: { type: 'number', minimum: 0 },
     maxBytes: { type: 'number', minimum: 0 },
+    maxBytesPerPartition: { type: 'number', minimum: 0 },
     maxWaitTime: { type: 'number', minimum: 0 },
     isolationLevel: { type: 'number', enum: allowedFetchIsolationLevels },
     deserializers: serdeProperties,

package/dist/clients/consumer/types.d.ts

@@ -65,6 +65,7 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
     autocommit?: boolean | number;
     minBytes?: number;
     maxBytes?: number;
+    maxBytesPerPartition?: number;
     maxWaitTime?: number;
     isolationLevel?: number;
     deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;

package/dist/clients/producer/partitioners.js

@@ -1,8 +1,10 @@
 import { murmur2 } from "../../protocol/murmur2.js";
 const compatibilityMurmur2Mask = 0x7fffffff;
 export function defaultPartitioner(_, key) {
+    /* c8 ignore next - Hard to test */
     return Buffer.isBuffer(key) ? murmur2(key) >>> 0 : 0;
 }
 export function compatibilityPartitioner(_, key) {
+    /* c8 ignore next - Hard to test */
     return Buffer.isBuffer(key) ? murmur2(key) & compatibilityMurmur2Mask : 0;
 }

package/dist/clients/producer/producer.d.ts

@@ -1,4 +1,5 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
+import { type Broker, type Connection } from '../../network/connection.ts';
 import { type Message } from '../../protocol/records.ts';
 import { kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator } from '../../symbols.ts';
 import { Base } from '../base/base.ts';
@@ -27,6 +28,11 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     asStream(options?: ProducerStreamOptions<Key, Value, HeaderKey, HeaderValue>): ProducerStream<Key, Value, HeaderKey, HeaderValue>;
     beginTransaction(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<Transaction<Key, Value, HeaderKey, HeaderValue>>): void;
     beginTransaction(options?: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<Transaction<Key, Value, HeaderKey, HeaderValue>>;
+    getSendTopicPartitions(options: SendOptions<Key, Value, HeaderKey, HeaderValue>): Map<string, Set<number>>;
+    getSendBrokers(options: SendOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<Record<string, Broker>>): void;
+    getSendBrokers(options: SendOptions<Key, Value, HeaderKey, HeaderValue>): Promise<Record<string, Broker>>;
+    getSendConnections(options: SendOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<Record<string, Connection>>): void;
+    getSendConnections(options: SendOptions<Key, Value, HeaderKey, HeaderValue>): Promise<Record<string, Connection>>;
     [kTransactionFindCoordinator](callback: CallbackWithPromise<void>): void;
     [kTransactionAddPartitions](transactionId: number, topicsPartitions: Map<string, Set<number>>, callback: CallbackWithPromise<void>): void;
     [kTransactionAddOffsets](transactionId: number, groupId: string, callback: CallbackWithPromise<void>): void;

package/dist/clients/producer/producer.js

@@ -243,6 +243,98 @@ export class Producer extends Base {
         producerTransactionsChannel.traceCallback(this.#transaction[kTransactionPrepare], 2, createDiagnosticContext({ client: this, operation: 'begin' }), this.#transaction, this.#producerInfo?.transactionalId, options, callback);
         return callback[kCallbackPromise];
     }
+    getSendTopicPartitions(options) {
+        const topicsPartitions = new Map();
+        const partitioner = options.partitioner ?? this[kOptions].partitioner;
+        for (const message of options.messages) {
+            const topic = message.topic;
+            let key;
+            let headers = new Map();
+            try {
+                if (message.headers) {
+                    headers =
+                        message.headers instanceof Map
+                            ? message.headers
+                            : new Map(Object.entries(message.headers));
+                }
+                key = this.#keySerializer(message.key, headers, message);
+            }
+            catch (error) {
+                throw new UserError('Failed to serialize a message.', { cause: error });
+            }
+            const partition = this.#assignPartition(message, partitioner, key, topic);
+            let partitions = topicsPartitions.get(topic);
+            if (!partitions) {
+                partitions = new Set();
+                topicsPartitions.set(topic, partitions);
+            }
+            partitions.add(partition);
+        }
+        return topicsPartitions;
+    }
+    getSendBrokers(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        let topicsPartitions;
+        try {
+            topicsPartitions = this.getSendTopicPartitions(options);
+        }
+        catch (error) {
+            callback(error);
+            return callback[kCallbackPromise];
+        }
+        this[kMetadata]({ topics: Array.from(topicsPartitions.keys()), autocreateTopics: options.autocreateTopics }, (error, metadata) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            const brokers = {};
+            for (const [topic, partitions] of topicsPartitions) {
+                for (const rawPartition of partitions) {
+                    const partition = rawPartition & metadata.topics.get(topic).partitionsCount;
+                    const leader = metadata.topics.get(topic).partitions[partition].leader;
+                    const broker = metadata.brokers.get(leader);
+                    brokers[`${topic}:${partition}`] = broker;
+                }
+            }
+            callback(null, brokers);
+        });
+        return callback[kCallbackPromise];
+    }
+    getSendConnections(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        this.getSendBrokers(options, (error, brokers) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            runConcurrentCallbacks('Preparing producer connections failed.', Object.entries(brokers), ([topicPartition, broker], concurrentCallback) => {
+                this[kGetConnection](broker, (error, connection) => {
+                    if (error) {
+                        concurrentCallback(error);
+                        return;
+                    }
+                    concurrentCallback(null, [topicPartition, connection]);
+                });
+            }, (error, results) => {
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                callback(null, Object.fromEntries(results));
+            });
+        });
+        return callback[kCallbackPromise];
+    }
     [kTransactionFindCoordinator](callback) {
         if (this.#coordinatorId !== undefined) {
             callback(null);
@@ -533,22 +625,7 @@ export class Producer extends Base {
                 callback(new UserError('Failed to serialize a message.', { cause: error }));
                 return;
             }
-            let partition = 0;
-            if (typeof message.partition !== 'number') {
-                if (partitioner) {
-                    partition = partitioner(message, key);
-                }
-                else if (key) {
-                    partition = defaultPartitioner(message, key);
-                }
-                else {
-                    // Use the roundrobin
-                    partition = this.#partitionsRoundRobin.postIncrement(topic, 1, 0);
-                }
-            }
-            else {
-                partition = message.partition;
-            }
+            const partition = this.#assignPartition(message, partitioner, key, topic);
             topics.add(topic);
             messages.push({
                 key,
@@ -610,16 +687,12 @@ export class Producer extends Base {
                 nodes.push(destination);
                 this.#performSingleDestinationSend(topics, destinationMessages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, concurrentCallback);
             }, (error, apiResults) => {
-                if (error) {
-                    callback(error);
-                    return;
-                }
                 this.#metricsProducedMessages?.inc(messages.length);
                 const results = {};
                 if (sendOptions.acks === ProduceAcks.NO_RESPONSE) {
                     const unwritableNodes = [];
                     for (let i = 0; i < apiResults.length; i++) {
-                        if (apiResults[i] === false) {
+                        if (typeof apiResults[i] === 'undefined' || apiResults[i] === false) {
                             unwritableNodes.push(nodes[i]);
                         }
                     }
@@ -628,7 +701,7 @@ export class Producer extends Base {
                 else {
                     const topics = [];
                     for (const result of apiResults) {
-                        for (const { name, partitionResponses } of result.responses) {
+                        for (const { name, partitionResponses } of result?.responses ?? []) {
                             for (const partitionResponse of partitionResponses) {
                                 topics.push({
                                     topic: name,
@@ -642,7 +715,14 @@ export class Producer extends Base {
                         results.offsets = topics;
                     }
                 }
-                callback(null, results);
+                if (error) {
+                    ;
+                    error.produced = results;
+                    callback(error);
+                }
+                else {
+                    callback(null, results);
+                }
             });
         });
     }
@@ -673,6 +753,7 @@ export class Producer extends Base {
                 if (error) {
                     // If the last error was due to stale metadata, we retry the operation with this set of messages
                     // since the partition is already set, it should attempt on the new destination
+                    /* c8 ignore next 3 - Hard to test */
                     const kafkaError = GenericError.isGenericError(error)
                         ? error
                         : null;
@@ -682,11 +763,12 @@ export class Producer extends Base {
                         this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, callback);
                         return;
                     }
-                    callback(error);
+                    callback(error, results);
                     return;
                 }
                 callback(error, results);
             }, 0, [], error => {
+                /* c8 ignore next 3 - Hard to test */
                 if (!repeatOnStaleMetadata || !GenericError.isGenericError(error)) {
                     return false;
                 }
@@ -752,4 +834,23 @@ export class Producer extends Base {
             this.#send(options, callback);
         });
     }
+    #assignPartition(message, partitioner, key, topic) {
+        let partition = 0;
+        if (typeof message.partition !== 'number') {
+            if (partitioner) {
+                partition = partitioner(message, key, { metadata: this.currentMetadata });
+            }
+            else if (key) {
+                partition = defaultPartitioner(message, key);
+            }
+            else {
+                // Use the roundrobin
+                partition = this.#partitionsRoundRobin.postIncrement(topic, 1, 0);
+            }
+        }
+        else {
+            partition = message.partition;
+        }
+        return partition;
+    }
 }

package/dist/clients/producer/types.d.ts

@@ -1,7 +1,7 @@
 import { type CompressionAlgorithmValue } from '../../protocol/compression.ts';
 import { type MessageToProduce } from '../../protocol/records.ts';
 import { type SchemaRegistry } from '../../registries/abstract.ts';
-import { type BaseOptions, type TopicWithPartitionAndOffset } from '../base/types.ts';
+import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from '../base/types.ts';
 import { type BeforeSerializationHook, type Serializers } from '../serde.ts';
 export interface ProducerInfo {
     producerId: bigint;
@@ -12,7 +12,10 @@ export interface ProduceResult {
     offsets?: TopicWithPartitionAndOffset[];
     unwritableNodes?: number[];
 }
-export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key?: Buffer | undefined) => number;
+export interface PartitionerContext {
+    metadata: ClusterMetadata | undefined;
+}
+export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key: Buffer | undefined, context: PartitionerContext) => number;
 export interface ProducerStreamReport {
     batchId: number;
     count: number;

package/dist/errors.js

@@ -120,7 +120,7 @@ export class ProtocolError extends GenericError {
                 'NOT_LEADER_OR_FOLLOWER',
                 'FENCED_LEADER_EPOCH'
             ].includes(id),
-            needsRejoin: ['MEMBER_ID_REQUIRED', 'UNKNOWN_MEMBER_ID', 'REBALANCE_IN_PROGRESS'].includes(id),
+            needsRejoin: ['MEMBER_ID_REQUIRED', 'UNKNOWN_MEMBER_ID', 'REBALANCE_IN_PROGRESS', 'ILLEGAL_GENERATION'].includes(id),
             producerFenced: id === 'INVALID_PRODUCER_EPOCH',
             rebalanceInProgress: id === 'REBALANCE_IN_PROGRESS',
             unknownMemberId: id === 'UNKNOWN_MEMBER_ID',

package/dist/network/connection-pool.d.ts

@@ -27,12 +27,15 @@ export declare class ConnectionPool extends TypedEventEmitter<ConnectionPoolEven
     get instanceId(): number;
     get ownerId(): number | undefined;
     get context(): unknown;
+    [Symbol.iterator](): MapIterator<[string, Connection]>;
     get(broker: Broker, callback: CallbackWithPromise<Connection>): void;
     get(broker: Broker): Promise<Connection>;
     getFirstAvailable(brokers: Broker[], callback: CallbackWithPromise<Connection>): void;
     getFirstAvailable(brokers: Broker[]): Promise<Connection>;
+    getEstablishedConnection(broker: Broker): Connection | undefined;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
     isActive(): boolean;
     isConnected(): boolean;
+    has(broker: Broker): boolean;
 }

package/dist/network/connection-pool.js

@@ -34,6 +34,9 @@ export class ConnectionPool extends TypedEventEmitter {
     get context() {
         return this.#context;
     }
+    [Symbol.iterator]() {
+        return this.#connections[Symbol.iterator]();
+    }
     get(broker, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -48,6 +51,9 @@ export class ConnectionPool extends TypedEventEmitter {
         connectionsPoolGetsChannel.traceCallback(this.#getFirstAvailable, 3, createDiagnosticContext({ connectionPool: this, brokers, operation: 'getFirstAvailable' }), this, brokers, 0, [], callback);
         return callback[kCallbackPromise];
     }
+    getEstablishedConnection(broker) {
+        return this.#connections.get(`${broker.host}:${broker.port}`);
+    }
     close(callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -81,6 +87,9 @@ export class ConnectionPool extends TypedEventEmitter {
         }
         return true;
     }
+    has(broker) {
+        return this.#connections.has(`${broker.host}:${broker.port}`);
+    }
     #get(broker, callback) {
         if (this.#closed) {
             callback(new Error('Connection pool is closed.'));

package/dist/network/connection.js

@@ -122,6 +122,7 @@ export class Connection extends TypedEventEmitter {
                 connectionOptions.servername =
                     typeof this.#options.tlsServerName === 'string' ? this.#options.tlsServerName : host;
             }
+            /* c8 ignore next 13 - Hard to test */
             const connectingSocketTimeoutHandler = () => {
                 const error = new TimeoutError(`Connection to ${host}:${port} timed out.`);
                 diagnosticContext.error = error;

package/dist/typescript-4/dist/clients/admin/admin.d.ts

@@ -5,7 +5,7 @@ import { type CallbackWithPromise } from "../../apis/callbacks";
 import { type Callback } from "../../apis/definitions";
 import { type Acl } from "../../apis/types";
 import { Base } from "../base/base";
-import { type AdminListOffsetsOptions, type AdminOptions, type AlterClientQuotasOptions, type AlterConfigsOptions, type AlterConsumerGroupOffsetsOptions, type BrokerLogDirDescription, type ConfigDescription, type CreateAclsOptions, type CreatedTopic, type CreatePartitionsOptions, type CreateTopicsOptions, type DeleteAclsOptions, type DeleteConsumerGroupOffsetsOptions, type DeleteRecordsOptions, type DeleteGroupsOptions, type DeleteTopicsOptions, type DeletedRecordsTopic, type DescribeAclsOptions, type DescribeClientQuotasOptions, type DescribeConfigsOptions, type DescribeGroupsOptions, type DescribeLogDirsOptions, type FindCoordinatorOptions, type FindCoordinatorResult, type Group, type GroupBase, type IncrementalAlterConfigsOptions, type ListConsumerGroupOffsetsGroup, type ListConsumerGroupOffsetsOptions, type ListedOffsetsTopic, type ListGroupsOptions, type ListTopicsOptions, type RemoveMembersFromConsumerGroupOptions } from "./types";
+import { type AdminListOffsetsOptions, type AdminOptions, type AlterClientQuotasOptions, type AlterConfigsOptions, type AlterConsumerGroupOffsetsOptions, type BrokerLogDirDescription, type ConfigDescription, type CreateAclsOptions, type CreatedTopic, type CreatePartitionsOptions, type CreateTopicsOptions, type DeleteAclsOptions, type DeleteConsumerGroupOffsetsOptions, type DeletedRecordsTopic, type DeleteGroupsOptions, type DeleteRecordsOptions, type DeleteTopicsOptions, type DescribeAclsOptions, type DescribeClientQuotasOptions, type DescribeConfigsOptions, type DescribeGroupsOptions, type DescribeLogDirsOptions, type FindCoordinatorOptions, type FindCoordinatorResult, type Group, type GroupBase, type IncrementalAlterConfigsOptions, type ListConsumerGroupOffsetsGroup, type ListConsumerGroupOffsetsOptions, type ListedOffsetsTopic, type ListGroupsOptions, type ListTopicsOptions, type RemoveMembersFromConsumerGroupOptions } from "./types";
 export declare class Admin extends Base<AdminOptions> {
   #private;
   constructor(options: AdminOptions);

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -66,6 +66,8 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   get closed(): boolean;
   get type(): ClientType;
   get context(): unknown;
+  get currentMetadata(): ClusterMetadata | undefined;
+  get connections(): ConnectionPool;
   emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
   close(callback: CallbackWithPromise<void>): void;
   close(): Promise<void>;

package/dist/typescript-4/dist/clients/consumer/messages-stream.d.ts

@@ -19,6 +19,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
   get offsetsToCommit(): Map<string, CommitOptionsPartition>;
   get offsetsCommitted(): Map<string, bigint>;
   get committedOffsets(): Map<string, bigint>;
+  get connections(): ConnectionPool;
   close(callback: CallbackWithPromise<void>): void;
   close(): Promise<void>;
   isActive(): boolean;

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -97,6 +97,10 @@ export declare const consumeOptionsProperties: {
     type: string;
     minimum: number;
   };
+  maxBytesPerPartition: {
+    type: string;
+    minimum: number;
+  };
   maxWaitTime: {
     type: string;
     minimum: number;
@@ -217,6 +221,10 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;
@@ -379,6 +387,10 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;
@@ -502,6 +514,10 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -65,6 +65,7 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
   autocommit?: boolean | number;
   minBytes?: number;
   maxBytes?: number;
+  maxBytesPerPartition?: number;
   maxWaitTime?: number;
   isolationLevel?: number;
   deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;

package/dist/typescript-4/dist/clients/producer/producer.d.ts

@@ -1,4 +1,5 @@
 import { type CallbackWithPromise } from "../../apis/callbacks";
+import { type Broker, type Connection } from "../../network/connection";
 import { type Message } from "../../protocol/records";
 import { kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator } from "../../symbols";
 import { Base } from "../base/base";
@@ -27,6 +28,11 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   asStream(options?: ProducerStreamOptions<Key, Value, HeaderKey, HeaderValue>): ProducerStream<Key, Value, HeaderKey, HeaderValue>;
   beginTransaction(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<Transaction<Key, Value, HeaderKey, HeaderValue>>): void;
   beginTransaction(options?: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<Transaction<Key, Value, HeaderKey, HeaderValue>>;
+  getSendTopicPartitions(options: SendOptions<Key, Value, HeaderKey, HeaderValue>): Map<string, Set<number>>;
+  getSendBrokers(options: SendOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<Record<string, Broker>>): void;
+  getSendBrokers(options: SendOptions<Key, Value, HeaderKey, HeaderValue>): Promise<Record<string, Broker>>;
+  getSendConnections(options: SendOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<Record<string, Connection>>): void;
+  getSendConnections(options: SendOptions<Key, Value, HeaderKey, HeaderValue>): Promise<Record<string, Connection>>;
   [kTransactionFindCoordinator](callback: CallbackWithPromise<void>): void;
   [kTransactionAddPartitions](transactionId: number, topicsPartitions: Map<string, Set<number>>, callback: CallbackWithPromise<void>): void;
   [kTransactionAddOffsets](transactionId: number, groupId: string, callback: CallbackWithPromise<void>): void;

package/dist/typescript-4/dist/clients/producer/types.d.ts

@@ -1,7 +1,7 @@
 import { type CompressionAlgorithmValue } from "../../protocol/compression";
 import { type MessageToProduce } from "../../protocol/records";
 import { type SchemaRegistry } from "../../registries/abstract";
-import { type BaseOptions, type TopicWithPartitionAndOffset } from "../base/types";
+import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
 import { type BeforeSerializationHook, type Serializers } from "../serde";
 export interface ProducerInfo {
   producerId: bigint;
@@ -12,7 +12,10 @@ export interface ProduceResult {
   offsets?: TopicWithPartitionAndOffset[];
   unwritableNodes?: number[];
 }
-export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key?: Buffer | undefined) => number;
+export interface PartitionerContext {
+  metadata: ClusterMetadata | undefined;
+}
+export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key: Buffer | undefined, context: PartitionerContext) => number;
 export interface ProducerStreamReport {
   batchId: number;
   count: number;

package/dist/typescript-4/dist/network/connection-pool.d.ts

@@ -27,12 +27,15 @@ export declare class ConnectionPool extends TypedEventEmitter<ConnectionPoolEven
   get instanceId(): number;
   get ownerId(): number | undefined;
   get context(): unknown;
+  [Symbol.iterator](): MapIterator<[string, Connection]>;
   get(broker: Broker, callback: CallbackWithPromise<Connection>): void;
   get(broker: Broker): Promise<Connection>;
   getFirstAvailable(brokers: Broker[], callback: CallbackWithPromise<Connection>): void;
   getFirstAvailable(brokers: Broker[]): Promise<Connection>;
+  getEstablishedConnection(broker: Broker): Connection | undefined;
   close(callback: CallbackWithPromise<void>): void;
   close(): Promise<void>;
   isActive(): boolean;
   isConnected(): boolean;
+  has(broker: Broker): boolean;
 }

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.33.2";
+export const version = "1.35.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.33.2",
+  "version": "1.35.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -24,6 +24,24 @@
   "type": "module",
   "exports": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "rm -rf dist && tsc -p tsconfig.base.json",
+    "postbuild": "./scripts/node scripts/postbuild.ts",
+    "lint": "eslint --cache",
+    "typecheck": "tsc -p . --noEmit",
+    "format": "prettier -w benchmarks playground src test",
+    "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:memory": "NODE_OPTIONS='--expose-gc' ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/**/*.memory-test.ts'",
+    "test:docker:up": "docker-compose up -d --wait",
+    "test:docker:down": "docker-compose down",
+    "ci": "npm run build && npm run lint && npm run test:ci",
+    "prepublishOnly": "npm run build && npm run lint",
+    "postpublish": "git push origin && git push origin -f --tags",
+    "generate:apis": "node scripts/generate-apis.ts",
+    "generate:errors": "node scripts/generate-errors.ts",
+    "create:api": "node scripts/create-api.ts"
+  },
   "dependencies": {
     "@platformatic/dynamic-buffer": "^0.3.1",
     "@platformatic/wasm-utils": "^0.2.1",
@@ -84,21 +102,5 @@
         "dist/typescript-4/*"
       ]
     }
-  },
-  "scripts": {
-    "build": "rm -rf dist && tsc -p tsconfig.base.json",
-    "postbuild": "./scripts/node scripts/postbuild.ts",
-    "lint": "eslint --cache",
-    "typecheck": "tsc -p . --noEmit",
-    "format": "prettier -w benchmarks playground src test",
-    "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
-    "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
-    "test:memory": "NODE_OPTIONS='--expose-gc' ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/**/*.memory-test.ts'",
-    "test:docker:up": "docker-compose up -d --wait",
-    "test:docker:down": "docker-compose down",
-    "ci": "npm run build && npm run lint && npm run test:ci",
-    "generate:apis": "node scripts/generate-apis.ts",
-    "generate:errors": "node scripts/generate-errors.ts",
-    "create:api": "node scripts/create-api.ts"
   }
 }