@platformatic/kafka 2.1.0 → 2.2.0

package/dist/clients/admin/admin.js

@@ -1,7 +1,7 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
-import { ConfigResourceTypes, FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
+import { ConfigResourceTypes, FetchIsolationLevels, FindCoordinatorKeyTypes, legacyConsumerGroupStates } from "../../apis/enumerations.js";
 import { adminAclsChannel, adminClientQuotasChannel, adminConfigsChannel, adminConsumerGroupOffsetsChannel, adminGroupsChannel, adminLogDirsChannel, adminOffsetsChannel, adminTopicsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { MultipleErrors, UserError } from "../../errors.js";
+import { findErrorBy, MultipleErrors, NetworkError, UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Base, kAfterCreate, kCheckNotClosed, kConnections, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
 import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteGroupsOptionsValidator, deleteRecordsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, findCoordinatorOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
@@ -366,15 +366,20 @@ export class Admin extends Base {
     }
     #getControllerConnection(callback) {
         if (this.#controller) {
-            this[kConnections].get(this.#controller, callback);
+            this[kConnections].get(this.#controller, (error, connection) => {
+                if (NetworkError.isRetryable(error)) {
+                    this.#controller = null;
+                }
+                callback(error, connection);
+            });
         }
         else {
             this[kGetBootstrapConnection](callback);
         }
     }
     #handleNotControllerError(error, value, callback) {
-        if (error && error?.findBy?.('apiCode', 41)) {
-            this.metadata({ topics: [] }, (metadataError, metadata) => {
+        if (findErrorBy(error, 'apiCode', 41)) {
+            this[kMetadata]({ topics: [], forceUpdate: true }, (metadataError, metadata) => {
                 /* c8 ignore next 4 - Hard to test */
                 if (metadataError) {
                     callback(metadataError);
@@ -407,7 +412,7 @@ export class Admin extends Base {
                 });
             }, (error, metadata) => {
                 if (error) {
-                    deduplicateCallback(error);
+                    deduplicateCallback(new MultipleErrors('Listing topics failed.', [error]));
                     return;
                 }
                 const topics = new Set();
@@ -422,26 +427,34 @@ export class Admin extends Base {
             }, 0);
         }, callback);
     }
+    #mapTopicAssignments(assignments) {
+        const mapped = [];
+        for (const { partition, brokers } of assignments ?? []) {
+            mapped.push({ partitionIndex: partition, brokerIds: brokers });
+        }
+        return mapped;
+    }
     #createTopics(options, callback) {
         // -1 is required if manual assignments are used. If no manual assignments are used, -1 will default to broker settings.
         const numPartitions = options.partitions ?? -1;
         const replicationFactor = options.replicas ?? -1;
-        const assignments = [];
+        const assignments = this.#mapTopicAssignments(options.assignments);
         const configs = options.configs ?? [];
-        for (const { partition, brokers } of options.assignments ?? []) {
-            assignments.push({ partitionIndex: partition, brokerIds: brokers });
-        }
         const requests = [];
         for (const topic of options.topics) {
+            if (typeof topic === 'string') {
+                requests.push({ name: topic, numPartitions, replicationFactor, assignments, configs });
+                continue;
+            }
             requests.push({
-                name: topic,
-                numPartitions,
-                replicationFactor,
-                assignments,
+                name: topic.topic,
+                numPartitions: topic.partitions ?? numPartitions,
+                replicationFactor: topic.replicas ?? replicationFactor,
+                assignments: topic.assignments ? this.#mapTopicAssignments(topic.assignments) : assignments,
                 configs
             });
         }
-        this[kPerformDeduplicated](`createTopics-${options.topics.join(',')}`, deduplicateCallback => {
+        this[kPerformDeduplicated](`createTopics-${options.topics.map(topic => (typeof topic === 'string' ? topic : topic.topic)).join(',')}`, deduplicateCallback => {
             this[kPerformWithRetry]('createTopics', retryCallback => {
                 this.#getControllerConnection((error, connection) => {
                     if (error) {
@@ -460,13 +473,14 @@ export class Admin extends Base {
                 });
             }, (error, response) => {
                 if (error) {
-                    deduplicateCallback(error);
+                    deduplicateCallback(new MultipleErrors('Creating topics failed.', [error]));
                     return;
                 }
                 const created = [];
-                for (const { name, topicId: id, numPartitions: partitions, replicationFactor: replicas, configs } of response.topics) {
+                const topics = response.topics;
+                for (const { name, topicId: id = '', numPartitions: partitions = -1, replicationFactor: replicas = -1, configs: topicConfigs } of topics) {
                     const configuration = {};
-                    for (const { name, value } of configs) {
+                    for (const { name, value } of topicConfigs ?? []) {
                         configuration[name] = value;
                     }
                     created.push({ id, name, partitions, replicas, configuration });
@@ -497,7 +511,13 @@ export class Admin extends Base {
                         });
                     });
                 });
-            }, deduplicateCallback, 0);
+            }, (error, response) => {
+                if (error) {
+                    deduplicateCallback(new MultipleErrors('Deleting topics failed.', [error]));
+                    return;
+                }
+                deduplicateCallback(null, response);
+            }, 0);
         }, error => callback(error));
     }
     #createPartitions(options, callback) {
@@ -529,13 +549,14 @@ export class Admin extends Base {
         });
     }
     #listGroups(options, callback) {
-        // Find all the brokers in the cluster
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
+        this[kMetadata]({ topics: [], forceUpdate: true }, (error, metadata) => {
             if (error) {
                 callback(error);
                 return;
             }
-            runConcurrentCallbacks('Listing groups failed.', metadata.brokers, ([, broker], concurrentCallback) => {
+            runConcurrentCallbacks('Listing groups failed.', 
+            // metadata must be defined as error was undefined
+            metadata.brokers, ([, broker], concurrentCallback) => {
                 this[kGetConnection](broker, (error, connection) => {
                     if (error) {
                         concurrentCallback(error);
@@ -547,8 +568,11 @@ export class Admin extends Base {
                                 retryCallback(error);
                                 return;
                             }
+                            // Brokers filter on the values they put on the wire, so the legacy Java enum
+                            // constant names have to be translated or they would silently match nothing.
+                            const states = (options.states ?? []).map(state => legacyConsumerGroupStates[state] ?? state);
                             /* c8 ignore next 5 */
-                            api(connection, options.states ?? [], options.types, retryCallback);
+                            api(connection, states, options.types, retryCallback);
                         });
                     }, concurrentCallback, 0);
                 });
@@ -562,8 +586,10 @@ export class Admin extends Base {
                     for (const raw of result.groups) {
                         groups.set(raw.groupId, {
                             id: raw.groupId,
-                            state: raw.groupState.toUpperCase(),
-                            groupType: raw.groupType,
+                            // ListGroups only carries group_state from v4, so below that the codec has nothing
+                            // to report. 'Unknown' is the value Kafka itself uses for exactly this case.
+                            state: (raw.groupState || 'Unknown'),
+                            groupType: raw.groupType || undefined,
                             protocolType: raw.protocolType
                         });
                     }
@@ -573,45 +599,42 @@ export class Admin extends Base {
         });
     }
     #describeGroups(options, callback) {
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
-            if (error) {
-                callback(error);
-                return;
-            }
+        this[kPerformWithRetry]('describeGroups', retryCallback => {
             this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: options.groups }, (error, coordinators) => {
                 if (error) {
-                    callback(error);
+                    // FindCoordinator already retried some times so no retry again
+                    callback(new MultipleErrors('Describing groups failed.', error instanceof MultipleErrors ? error.errors : [error]));
                     return;
                 }
-                // Group the groups by coordinator
                 const coordinatorsMap = new Map();
-                for (const { key: group, nodeId: node } of coordinators) {
-                    let coordinator = coordinatorsMap.get(node);
-                    if (!coordinator) {
-                        coordinator = [];
-                        coordinatorsMap.set(node, coordinator);
+                // coordinators must be defined as error was undefined
+                for (const coordinator of coordinators) {
+                    let coordinatorAggregate = coordinatorsMap.get(coordinator.nodeId);
+                    if (!coordinatorAggregate) {
+                        coordinatorAggregate = { host: coordinator.host, port: coordinator.port, groups: [] };
+                        coordinatorsMap.set(coordinator.nodeId, coordinatorAggregate);
                     }
-                    coordinator.push(group);
+                    coordinatorAggregate.groups.push(coordinator.key);
                 }
-                runConcurrentCallbacks('Describing groups failed.', coordinatorsMap, ([node, groups], concurrentCallback) => {
-                    this[kGetConnection](metadata.brokers.get(node), (error, connection) => {
-                        if (error) {
-                            concurrentCallback(error);
-                            return;
-                        }
-                        this[kPerformWithRetry]('describeGroups', retryCallback => {
+                runConcurrentCallbacks('Describing groups failed.', coordinatorsMap.values(), (coordinator, concurrentCallback) => {
+                    this[kPerformWithRetry]('describeGroupsOnCoordinator', retryCallback => {
+                        this[kGetConnection](coordinator, (error, connection) => {
+                            if (error) {
+                                retryCallback(error);
+                                return;
+                            }
                             this[kGetApi]('DescribeGroups', (error, api) => {
                                 if (error) {
                                     retryCallback(error);
                                     return;
                                 }
-                                api(connection, groups, options.includeAuthorizedOperations ?? false, retryCallback);
+                                api(connection, coordinator.groups, options.includeAuthorizedOperations ?? false, retryCallback);
                             });
-                        }, concurrentCallback, 0);
-                    });
+                        });
+                    }, concurrentCallback, 0);
                 }, (error, results) => {
                     if (error) {
-                        callback(error);
+                        retryCallback(error);
                         return;
                     }
                     const groups = new Map();
@@ -619,7 +642,7 @@ export class Admin extends Base {
                         for (const raw of result.groups) {
                             const group = {
                                 id: raw.groupId,
-                                state: raw.groupState.toUpperCase(),
+                                state: raw.groupState,
                                 protocolType: raw.protocolType,
                                 protocol: raw.protocolData,
                                 members: new Map(),
@@ -655,59 +678,58 @@ export class Admin extends Base {
                             groups.set(group.id, group);
                         }
                     }
-                    callback(null, groups);
+                    retryCallback(null, groups);
                 });
             });
-        });
+        }, callback, 0);
     }
     #deleteGroups(options, callback) {
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
-            if (error) {
-                callback(error);
-                return;
-            }
+        this[kPerformWithRetry]('deleteGroups', retryCallback => {
             this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: options.groups }, (error, coordinators) => {
                 if (error) {
-                    callback(error);
+                    // FindCoordinator already retried some times so no retry again
+                    callback(new MultipleErrors('Deleting groups failed.', error instanceof MultipleErrors ? error.errors : [error]));
                     return;
                 }
-                // Group the groups by coordinator
                 const coordinatorsMap = new Map();
-                for (const { key: group, nodeId: node } of coordinators) {
-                    let coordinator = coordinatorsMap.get(node);
-                    if (!coordinator) {
-                        coordinator = [];
-                        coordinatorsMap.set(node, coordinator);
+                // coordinators must be defined as error was undefined
+                for (const coordinator of coordinators) {
+                    let coordinatorAggregate = coordinatorsMap.get(coordinator.nodeId);
+                    if (!coordinatorAggregate) {
+                        coordinatorAggregate = { host: coordinator.host, port: coordinator.port, groups: [] };
+                        coordinatorsMap.set(coordinator.nodeId, coordinatorAggregate);
                     }
-                    coordinator.push(group);
+                    coordinatorAggregate.groups.push(coordinator.key);
                 }
-                runConcurrentCallbacks('Deleting groups failed.', coordinatorsMap, ([node, groups], concurrentCallback) => {
-                    this[kGetConnection](metadata.brokers.get(node), (error, connection) => {
-                        if (error) {
-                            concurrentCallback(error);
-                            return;
-                        }
-                        this[kPerformWithRetry]('deleteGroups', retryCallback => {
+                runConcurrentCallbacks('Deleting groups failed.', coordinatorsMap.values(), (coordinator, concurrentCallback) => {
+                    this[kPerformWithRetry]('deleteGroupsOnCoordinator', retryCallback => {
+                        this[kGetConnection](coordinator, (error, connection) => {
+                            if (error) {
+                                retryCallback(error);
+                                return;
+                            }
                             this[kGetApi]('DeleteGroups', (error, api) => {
                                 if (error) {
                                     retryCallback(error);
                                     return;
                                 }
-                                api(connection, groups, retryCallback);
+                                api(connection, coordinator.groups, retryCallback);
                             });
-                        }, concurrentCallback, 0);
-                    });
-                }, error => callback(error));
+                        });
+                    }, concurrentCallback, 0);
+                }, retryCallback);
             });
-        });
+        }, callback, 0);
     }
     #removeMembersFromConsumerGroup(options, callback) {
         if (!options.members || options.members.length === 0) {
+            // Remove all members if none are specified
             this.#describeGroups({ groups: [options.groupId] }, (error, groupsMap) => {
                 if (error) {
                     callback(new MultipleErrors('Removing members from consumer group failed.', [error]));
                     return;
                 }
+                // groupsMap must be defined as error was undefined
                 const group = groupsMap.get(options.groupId);
                 /* c8 ignore next 4 - Hard to test */
                 if (!group) {
@@ -724,61 +746,52 @@ export class Admin extends Base {
             });
             return;
         }
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
-            if (error) {
-                callback(error, undefined);
-                return;
-            }
+        this[kPerformWithRetry]('removeMembersFromConsumerGroup', retryCallback => {
             this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: [options.groupId] }, (error, coordinators) => {
                 if (error) {
-                    callback(new MultipleErrors('Removing members from consumer group failed.', [error]));
+                    // FindCoordinator already retried some times so no retry again
+                    callback(new MultipleErrors('Removing members from consumer group failed.', error instanceof MultipleErrors ? error.errors : [error]));
                     return;
                 }
                 const coordinator = coordinators.find(c => c.key === options.groupId);
                 /* c8 ignore next 8 - Hard to test */
                 if (!coordinator) {
-                    callback(new MultipleErrors('Removing members from consumer group failed.', [
-                        new Error(`No coordinator found for group ${options.groupId}`)
-                    ]));
-                    return;
-                }
-                const broker = metadata.brokers.get(coordinator.nodeId);
-                /* c8 ignore next 8 - Hard to test */
-                if (!broker) {
-                    callback(new MultipleErrors('Removing members from consumer group failed.', [
-                        new Error(`Broker ${coordinator.nodeId} not found`)
-                    ]));
+                    retryCallback(new Error('Group coordinator not found'));
                     return;
                 }
-                this[kGetConnection](broker, (error, connection) => {
+                this[kGetConnection](coordinator, (error, connection) => {
                     if (error) {
-                        callback(new MultipleErrors('Removing members from consumer group failed.', [error]));
+                        retryCallback(error);
                         return;
                     }
-                    this[kPerformWithRetry]('removeMembersFromConsumerGroup', retryCallback => {
-                        this[kGetApi]('LeaveGroup', (error, api) => {
-                            if (error) {
-                                retryCallback(error);
-                                return;
-                            }
-                            /* c8 ignore next 5 - Hard to test */
-                            const members = options.members.map(member => ({
-                                memberId: typeof member === 'string' ? member : member.memberId,
-                                groupInstanceId: null,
-                                reason: typeof member === 'string' ? 'Not specified' : member.reason
-                            }));
-                            api(connection, options.groupId, members, retryCallback);
-                        });
-                    }, error => {
+                    this[kGetApi]('LeaveGroup', (error, api) => {
                         if (error) {
-                            callback(new MultipleErrors('Removing members from consumer group failed.', [error]));
+                            retryCallback(error);
+                            return;
+                        }
+                        /* c8 ignore next 5 - Hard to test */
+                        const members = options.members.map(member => ({
+                            memberId: typeof member === 'string' ? member : member.memberId,
+                            groupInstanceId: null,
+                            reason: typeof member === 'string' ? 'Not specified' : member.reason
+                        }));
+                        if (api.version >= 3) {
+                            api(connection, options.groupId, members, retryCallback);
                             return;
                         }
-                        callback(null);
-                    }, 0);
+                        runConcurrentCallbacks('Removing members from consumer group failed.', members, (member, concurrentCallback) => {
+                            api(connection, options.groupId, [member], concurrentCallback);
+                        }, error => retryCallback(error));
+                    });
                 });
             });
-        });
+        }, error => {
+            if (error) {
+                callback(new MultipleErrors('Removing members from consumer group failed.', [error]));
+                return;
+            }
+            callback(null);
+        }, 0);
     }
     #findCoordinator(options, callback) {
         this[kPerformWithRetry]('findCoordinator', retryCallback => {
@@ -792,12 +805,33 @@ export class Admin extends Base {
                         retryCallback(error);
                         return;
                     }
-                    api(connection, options.keyType, options.keys, retryCallback);
+                    if (api.version >= 4) {
+                        api(connection, options.keyType, options.keys, retryCallback);
+                        return;
+                    }
+                    runConcurrentCallbacks('Finding coordinator failed.', options.keys, (key, concurrentCallback) => {
+                        api(connection, options.keyType, [key], (error, response) => {
+                            if (response) {
+                                response.coordinators[0].key = key;
+                            }
+                            concurrentCallback(error, response);
+                        });
+                    }, (error, responses) => {
+                        if (error) {
+                            retryCallback(error);
+                            return;
+                        }
+                        retryCallback(null, {
+                            // The caller controls how many keys are looked up, so reduce rather than spread.
+                            throttleTimeMs: responses.reduce((max, response) => Math.max(max, response.throttleTimeMs), 0),
+                            coordinators: responses.flatMap(response => response.coordinators)
+                        });
+                    });
                 });
             });
         }, (error, response) => {
             if (error) {
-                callback(error);
+                callback(new MultipleErrors('Finding coordinator failed.', [error]));
                 return;
             }
             callback(null, response.coordinators.map(coordinator => ({
@@ -828,7 +862,7 @@ export class Admin extends Base {
                 callback(new MultipleErrors('Describing client quotas failed.', [error]));
                 return;
             }
-            callback(null, response.entries);
+            callback(null, response.entries ?? []);
         }, 0);
     }
     #alterClientQuotas(options, callback) {
@@ -855,19 +889,21 @@ export class Admin extends Base {
         }, 0);
     }
     #describeLogDirs(options, callback) {
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
+        this[kMetadata]({ topics: [], forceUpdate: true }, (error, metadata) => {
             /* c8 ignore next 4 - Hard to test */
             if (error) {
                 callback(error);
                 return;
             }
-            runConcurrentCallbacks('Describing log dirs failed.', metadata.brokers, ([id, broker], concurrentCallback) => {
-                this[kGetConnection](broker, (error, connection) => {
-                    if (error) {
-                        concurrentCallback(error);
-                        return;
-                    }
-                    this[kPerformWithRetry]('describeLogDirs', retryCallback => {
+            runConcurrentCallbacks('Describing log dirs failed.', 
+            // metadata must be defined as error was undefined
+            metadata.brokers, ([id, broker], concurrentCallback) => {
+                this[kPerformWithRetry]('describeLogDirs', retryCallback => {
+                    this[kGetConnection](broker, (error, connection) => {
+                        if (error) {
+                            retryCallback(error);
+                            return;
+                        }
                         this[kGetApi]('DescribeLogDirs', (error, api) => {
                             if (error) {
                                 retryCallback(error);
@@ -875,82 +911,114 @@ export class Admin extends Base {
                             }
                             api(connection, options.topics, retryCallback);
                         });
-                    }, (error, response) => {
-                        if (error) {
-                            concurrentCallback(error);
-                            return;
-                        }
-                        concurrentCallback(null, {
-                            broker: id,
-                            throttleTimeMs: response.throttleTimeMs,
-                            results: response.results.map(result => ({
-                                logDir: result.logDir,
-                                topics: result.topics,
-                                totalBytes: result.totalBytes,
-                                usableBytes: result.usableBytes
-                            }))
-                        });
-                    }, 0);
-                });
+                    });
+                }, (error, response) => {
+                    if (error) {
+                        concurrentCallback(error);
+                        return;
+                    }
+                    concurrentCallback(null, {
+                        broker: id,
+                        throttleTimeMs: response.throttleTimeMs,
+                        results: response.results.map(result => ({
+                            logDir: result.logDir,
+                            topics: result.topics,
+                            totalBytes: result.totalBytes,
+                            usableBytes: result.usableBytes
+                        }))
+                    });
+                }, 0);
             }, callback);
         });
     }
     #listConsumerGroupOffsets(options, callback) {
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
-            if (error) {
-                callback(error);
-                return;
-            }
+        this[kPerformWithRetry]('listConsumerGroupOffsets', retryCallback => {
             /* c8 ignore next - Hard to test */
             const groupIds = options.groups.map(group => (typeof group === 'string' ? group : group.groupId));
             this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: groupIds }, (error, coordinators) => {
                 if (error) {
-                    callback(new MultipleErrors('Listing consumer group offsets failed.', [error]));
+                    // FindCoordinator already retried some times so no retry again
+                    callback(new MultipleErrors('Listing consumer group offsets failed.', error instanceof MultipleErrors ? error.errors : [error]));
                     return;
                 }
                 const coordinatorsMap = new Map();
-                for (const { key: groupId, nodeId: node } of coordinators) {
+                for (const coordinator of coordinators) {
                     const groupRequest = {
-                        groupId,
+                        groupId: coordinator.key,
                         memberId: null,
                         memberEpoch: -1
                     };
                     for (const group of options.groups) {
-                        if (typeof group !== 'string' && group.groupId === groupId && !!group.topics && group.topics.length > 0) {
+                        if (typeof group !== 'string' &&
+                            group.groupId === coordinator.key &&
+                            !!group.topics &&
+                            group.topics.length > 0) {
                             groupRequest.topics = group.topics;
                             break;
                         }
                     }
-                    let coordinator = coordinatorsMap.get(node);
-                    if (!coordinator) {
-                        coordinator = [];
-                        coordinatorsMap.set(node, coordinator);
+                    let offsetFetchRequestGroups = coordinatorsMap.get(coordinator.nodeId);
+                    if (!offsetFetchRequestGroups) {
+                        offsetFetchRequestGroups = { coordinator, groupRequests: [] };
+                        coordinatorsMap.set(coordinator.nodeId, offsetFetchRequestGroups);
                     }
-                    coordinator.push(groupRequest);
+                    offsetFetchRequestGroups.groupRequests.push(groupRequest);
                 }
-                runConcurrentCallbacks('Listing consumer group offsets failed.', coordinatorsMap, ([node, groups], concurrentCallback) => {
-                    this[kGetConnection](metadata.brokers.get(node), (error, connection) => {
-                        if (error) {
-                            concurrentCallback(error);
-                            return;
-                        }
-                        this[kPerformWithRetry]('offsetFetch', retryCallback => {
+                runConcurrentCallbacks('Listing consumer group offsets failed.', coordinatorsMap.values(), ({ coordinator, groupRequests }, concurrentCallback) => {
+                    this[kPerformWithRetry]('offsetFetch', retryCallback => {
+                        this[kGetConnection](coordinator, (error, connection) => {
+                            if (error) {
+                                retryCallback(error);
+                                return;
+                            }
                             this[kGetApi]('OffsetFetch', (error, api) => {
                                 if (error) {
                                     retryCallback(error);
                                     return;
                                 }
-                                /* c8 ignore next - Hard to test */
-                                api(connection, groups, options.requireStable ?? false, retryCallback);
+                                if (api.version >= 8) {
+                                    api(connection, groupRequests, options.requireStable ?? false, retryCallback);
+                                    return;
+                                }
+                                runConcurrentCallbacks('Listing consumer group offsets failed.', groupRequests, (groupRequest, groupCallback) => {
+                                    api(connection, [groupRequest], options.requireStable ?? false, (error, response) => {
+                                        if (error) {
+                                            groupCallback(error);
+                                            return;
+                                        }
+                                        const legacyResponse = response;
+                                        groupCallback(null, {
+                                            throttleTimeMs: legacyResponse.throttleTimeMs,
+                                            groups: [
+                                                {
+                                                    groupId: groupRequest.groupId,
+                                                    topics: legacyResponse.topics,
+                                                    errorCode: legacyResponse.errorCode
+                                                }
+                                            ]
+                                        });
+                                    });
+                                }, (error, responses) => {
+                                    if (error) {
+                                        retryCallback(error);
+                                        return;
+                                    }
+                                    retryCallback(null, {
+                                        throttleTimeMs: responses.reduce((max, response) => Math.max(max, response.throttleTimeMs), 0),
+                                        groups: responses.flatMap(response => response.groups)
+                                    });
+                                });
                             });
-                        }, concurrentCallback, 0);
-                    });
+                        });
+                    }, concurrentCallback, 0);
                 }, (error, responses) => {
                     if (error) {
-                        callback(error);
+                        retryCallback(error);
                         return;
                     }
-                    callback(null, responses.flatMap(r => r.groups.map(group => ({
+                    retryCallback(null, 
+                    // responses must be defined as error was undefined
+                    responses.flatMap(r => r.groups.map(group => ({
                         groupId: group.groupId,
                         topics: group.topics.map(topic => ({
                             name: topic.name,
@@ -964,125 +1032,101 @@ export class Admin extends Base {
                     }))));
                 });
             });
-        });
+        }, callback, 0);
     }
     #alterConsumerGroupOffsets(options, callback) {
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
-            if (error) {
-                callback(error, undefined);
-                return;
-            }
+        this[kPerformWithRetry]('alterConsumerGroupOffsets', retryCallback => {
             this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: [options.groupId] }, (error, coordinators) => {
                 if (error) {
-                    callback(new MultipleErrors('Altering consumer group offsets failed.', [error]));
+                    // FindCoordinator already retried some times so no retry again
+                    callback(new MultipleErrors('Altering consumer group offsets failed.', error instanceof MultipleErrors ? error.errors : [error]));
                     return;
                 }
                 const coordinator = coordinators.find(c => c.key === options.groupId);
                 /* c8 ignore next 9 - Hard to test */
                 if (!coordinator) {
-                    callback(new MultipleErrors('Altering consumer group offsets failed.', [
-                        new Error(`No coordinator found for group ${options.groupId}`)
-                    ]));
+                    retryCallback(new Error('Group coordinator not found'));
                     return;
                 }
-                const broker = metadata.brokers.get(coordinator.nodeId);
-                /* c8 ignore next 9 - Hard to test */
-                if (!broker) {
-                    callback(new MultipleErrors('Altering consumer group offsets failed.', [
-                        new Error(`Broker ${coordinator.nodeId} not found`)
-                    ]));
-                    return;
-                }
-                this[kGetConnection](broker, (error, connection) => {
+                this[kGetConnection](coordinator, (error, connection) => {
                     if (error) {
-                        callback(new MultipleErrors('Altering consumer group offsets failed.', [error]));
+                        retryCallback(error);
                         return;
                     }
-                    this[kPerformWithRetry]('alterConsumerGroupOffsets', retryCallback => {
-                        this[kGetApi]('OffsetCommit', (error, api) => {
-                            if (error) {
-                                retryCallback(error);
-                                return;
-                            }
-                            const topics = [];
-                            for (const topic of options.topics) {
-                                const partitions = topic.partitionOffsets.map(p => ({
-                                    partitionIndex: p.partition,
-                                    committedOffset: p.offset,
-                                    committedLeaderEpoch: -1,
-                                    committedMetadata: null
-                                }));
-                                topics.push({ name: topic.name, partitions });
-                            }
-                            api(connection, options.groupId, -1, '', null, topics, retryCallback);
-                        });
-                    }, error => {
+                    this[kGetApi]('OffsetCommit', (error, api) => {
                         if (error) {
-                            callback(new MultipleErrors('Altering consumer group offsets failed.', [error]));
+                            retryCallback(error);
                             return;
                         }
-                        callback(null);
-                    }, 0);
+                        const topics = [];
+                        for (const topic of options.topics) {
+                            const partitions = topic.partitionOffsets.map(p => ({
+                                partitionIndex: p.partition,
+                                committedOffset: p.offset,
+                                committedLeaderEpoch: -1,
+                                committedMetadata: null
+                            }));
+                            topics.push({ name: topic.name, partitions });
+                        }
+                        api(connection, options.groupId, -1, '', null, topics, retryCallback);
+                    });
                 });
             });
-        });
-    }
-    #deleteConsumerGroupOffsets(options, callback) {
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
+        }, error => {
             if (error) {
-                callback(error);
+                callback(new MultipleErrors('Altering consumer group offsets failed.', [error]));
                 return;
             }
+            callback(null);
+        }, 0);
+    }
+    #deleteConsumerGroupOffsets(options, callback) {
+        this[kPerformWithRetry]('deleteConsumerGroupOffsets', retryCallback => {
             this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: [options.groupId] }, (error, coordinators) => {
                 if (error) {
-                    callback(new MultipleErrors('Deleting consumer group offsets failed.', [error]));
+                    // FindCoordinator already retried some times so no retry again
+                    callback(new MultipleErrors('Deleting consumer group offsets failed.', error instanceof MultipleErrors ? error.errors : [error]));
                     return;
                 }
                 const coordinator = coordinators.find(c => c.key === options.groupId);
                 /* c8 ignore next 9 - Hard to test */
                 if (!coordinator) {
-                    callback(new MultipleErrors('Deleting consumer group offsets failed.', [
-                        new Error(`No coordinator found for group ${options.groupId}`)
-                    ]));
+                    retryCallback(new Error('Group coordinator not found'));
                     return;
                 }
-                const broker = metadata.brokers.get(coordinator.nodeId);
-                /* c8 ignore next 9 - Hard to test */
-                if (!broker) {
-                    callback(new MultipleErrors('Deleting consumer group offsets failed.', [
-                        new Error(`Broker ${coordinator.nodeId} not found`)
-                    ]));
-                    return;
-                }
-                this[kGetConnection](broker, (error, connection) => {
+                this[kGetConnection](coordinator, (error, connection) => {
                     if (error) {
-                        callback(new MultipleErrors('Deleting consumer group offsets failed.', [error]));
+                        retryCallback(error);
                         return;
                     }
-                    this[kPerformWithRetry]('deleteOffset', retryCallback => {
-                        this[kGetApi]('OffsetDelete', (error, api) => {
+                    this[kGetApi]('OffsetDelete', (error, api) => {
+                        if (error) {
+                            retryCallback(error);
+                            return;
+                        }
+                        api(connection, options.groupId, options.topics.map(t => ({
+                            name: t.name,
+                            partitions: t.partitionIndexes.map(p => ({ partitionIndex: p }))
+                        })), (error, response) => {
                             if (error) {
                                 retryCallback(error);
                                 return;
                             }
-                            api(connection, options.groupId, options.topics.map(t => ({
-                                name: t.name,
-                                partitions: t.partitionIndexes.map(p => ({ partitionIndex: p }))
-                            })), retryCallback);
+                            retryCallback(null, response.topics.map(topic => ({
+                                name: topic.name,
+                                partitionIndexes: topic.partitions.map(p => p.partitionIndex)
+                            })));
                         });
-                    }, (error, response) => {
-                        if (error) {
-                            callback(new MultipleErrors('Deleting consumer group offsets failed.', [error]));
-                            return;
-                        }
-                        callback(null, response.topics.map(topic => ({
-                            name: topic.name,
-                            partitionIndexes: topic.partitions.map(p => p.partitionIndex)
-                        })));
-                    }, 0);
+                    });
                 });
             });
-        });
+        }, (error, results) => {
+            if (error) {
+                callback(new MultipleErrors('Deleting consumer group offsets failed.', [error]));
+                return;
+            }
+            callback(null, results);
+        }, 0);
     }
     #getConfigRequestsDistributedToBrokers(resources) {
         const brokerConfigResourceMap = new Map();
@@ -1108,7 +1152,7 @@ export class Admin extends Base {
             this[kGetBootstrapConnection](callback);
             return;
         }
-        this[kMetadata]({ topics: [] }, (error, metadata) => {
+        this[kMetadata]({ topics: [], forceUpdate: true }, (error, metadata) => {
             /* c8 ignore next 4 - Hard to test */
             if (error) {
                 callback(error);
@@ -1125,13 +1169,7 @@ export class Admin extends Base {
     }
     #describeConfigs(options, callback) {
         runConcurrentCallbacks('Describing configs failed.', this.#getConfigRequestsDistributedToBrokers(options.resources), ([brokerId, resources], concurrentCallback) => {
-            this.#describeConfigsOnBroker({ ...options, resources }, brokerId, (error, response) => {
-                if (error) {
-                    concurrentCallback(error);
-                    return;
-                }
-                concurrentCallback(null, response);
-            });
+            this.#describeConfigsOnBroker({ ...options, resources }, brokerId, concurrentCallback);
         }, (error, results) => {
             callback(error, results?.flat());
         });
@@ -1307,154 +1345,166 @@ export class Admin extends Base {
         }, 0);
     }
     #deleteRecords(options, callback) {
-        this[kMetadata]({ topics: options.topics.map(topic => topic.name) }, (error, metadata) => {
-            if (error) {
-                callback(error);
-                return;
-            }
-            const requestsByLeader = new Map();
-            for (const topic of options.topics) {
-                for (const partition of topic.partitions) {
-                    const { leader } = metadata.topics.get(topic.name).partitions[partition.partition];
-                    let leaderRequests = requestsByLeader.get(leader);
-                    if (!leaderRequests) {
-                        leaderRequests = new Map();
-                        requestsByLeader.set(leader, leaderRequests);
-                    }
-                    let topicRequest = leaderRequests.get(topic.name);
-                    if (!topicRequest) {
-                        topicRequest = { name: topic.name, partitions: [] };
-                        leaderRequests.set(topic.name, topicRequest);
-                    }
-                    topicRequest.partitions.push({
-                        partitionIndex: partition.partition,
-                        offset: partition.offset
-                    });
+        this[kPerformWithRetry]('deleteRecords', retryCallback => {
+            this[kMetadata]({ topics: options.topics.map(topic => topic.name), forceUpdate: true }, (error, metadata) => {
+                if (error) {
+                    // Metadata request already retried some times so no retry again
+                    callback(error);
+                    return;
                 }
-            }
-            const requests = new Map();
-            for (const [leader, topics] of requestsByLeader) {
-                requests.set(leader, Array.from(topics.values()));
-            }
-            runConcurrentCallbacks('Deleting records failed.', requests, ([leader, requests], concurrentCallback) => {
-                this[kGetConnection](metadata.brokers.get(leader), (error, connection) => {
-                    if (error) {
-                        concurrentCallback(error);
-                        return;
+                const requests = new Map();
+                for (const topic of options.topics) {
+                    for (const partition of topic.partitions) {
+                        // metadata!.topics.get(topic.name) must be defined as the metadata request was successful
+                        const topicData = metadata.topics.get(topic.name);
+                        const targetPartitionData = topicData.partitions[partition.partition];
+                        /* c8 ignore next 4 - Hard to test */
+                        if (!targetPartitionData) {
+                            retryCallback(new UserError('Unknown partition.'));
+                            return;
+                        }
+                        const broker = metadata.brokers.get(targetPartitionData.leader);
+                        let leaderRequests = requests.get(broker);
+                        if (!leaderRequests) {
+                            leaderRequests = [];
+                            requests.set(broker, leaderRequests);
+                        }
+                        let topicRequest = leaderRequests.find(tr => tr.name === topic.name);
+                        if (!topicRequest) {
+                            topicRequest = { name: topic.name, partitions: [] };
+                            leaderRequests.push(topicRequest);
+                        }
+                        topicRequest.partitions.push({
+                            partitionIndex: partition.partition,
+                            offset: partition.offset
+                        });
                     }
-                    this[kPerformWithRetry]('deleteRecords', retryCallback => {
-                        this[kGetApi]('DeleteRecords', (error, api) => {
+                }
+                runConcurrentCallbacks('Deleting records failed.', requests, ([leader, requests], concurrentCallback) => {
+                    this[kPerformWithRetry]('deleteRecordsOnLeader', retryCallback => {
+                        this[kGetConnection](leader, (error, connection) => {
                             if (error) {
                                 retryCallback(error);
                                 return;
                             }
-                            api(connection, Array.from(requests.values()), this[kOptions].timeout, retryCallback);
+                            this[kGetApi]('DeleteRecords', (error, api) => {
+                                if (error) {
+                                    retryCallback(error);
+                                    return;
+                                }
+                                api(connection, requests, this[kOptions].timeout, retryCallback);
+                            });
                         });
                     }, concurrentCallback, 0);
+                }, (error, responses) => {
+                    if (error) {
+                        // Do not retry failed deletion requests
+                        callback(error);
+                        return;
+                    }
+                    const deletedRecordsByTopic = new Map();
+                    for (const response of responses) {
+                        for (const topic of response.topics) {
+                            let topicDeletedRecords = deletedRecordsByTopic.get(topic.name);
+                            if (!topicDeletedRecords) {
+                                topicDeletedRecords = { name: topic.name, partitions: [] };
+                                deletedRecordsByTopic.set(topic.name, topicDeletedRecords);
+                            }
+                            for (const partition of topic.partitions) {
+                                topicDeletedRecords.partitions.push({
+                                    partition: partition.partitionIndex,
+                                    lowWatermark: partition.lowWatermark
+                                });
+                            }
+                        }
+                    }
+                    retryCallback(null, Array.from(deletedRecordsByTopic.values()));
                 });
-            }, (error, responses) => {
+            });
+        }, callback, 0);
+    }
+    #listOffsets(options, callback) {
+        this[kPerformWithRetry]('listOffsets', retryCallback => {
+            this[kMetadata]({ topics: options.topics.map(topic => topic.name), forceUpdate: true }, (error, metadata) => {
                 if (error) {
+                    // Metadata request already retried some times so no retry again
                     callback(error);
                     return;
                 }
-                const deletedRecordsByTopic = new Map();
-                for (const response of responses) {
-                    for (const topic of response.topics) {
-                        let topicDeletedRecords = deletedRecordsByTopic.get(topic.name);
-                        if (!topicDeletedRecords) {
-                            topicDeletedRecords = { name: topic.name, partitions: [] };
-                            deletedRecordsByTopic.set(topic.name, topicDeletedRecords);
+                // metadata must be defined at this point
+                const { topics, brokers } = metadata;
+                const requests = new Map();
+                for (const topic of options.topics) {
+                    for (const partition of topic.partitions) {
+                        // topics.get(topic.name) must be defined as the metadata request was successful
+                        const topicData = topics.get(topic.name);
+                        const targetPartitionData = topicData.partitions[partition.partitionIndex];
+                        /* c8 ignore next 4 - Hard to test */
+                        if (!targetPartitionData) {
+                            retryCallback(new UserError('Unknown partition.'));
+                            return;
                         }
-                        for (const partition of topic.partitions) {
-                            topicDeletedRecords.partitions.push({
-                                partition: partition.partitionIndex,
-                                lowWatermark: partition.lowWatermark
-                            });
+                        const broker = brokers.get(targetPartitionData.leader);
+                        let leaderRequests = requests.get(broker);
+                        if (!leaderRequests) {
+                            leaderRequests = [];
+                            requests.set(broker, leaderRequests);
                         }
+                        let topicRequest = leaderRequests.find(t => t.name === topic.name);
+                        if (!topicRequest) {
+                            topicRequest = { name: topic.name, partitions: [] };
+                            leaderRequests.push(topicRequest);
+                        }
+                        topicRequest.partitions.push({
+                            partitionIndex: partition.partitionIndex,
+                            currentLeaderEpoch: targetPartitionData.leaderEpoch,
+                            /* c8 ignore next - Hard to test */
+                            timestamp: partition.timestamp ?? -1n
+                        });
                     }
                 }
-                callback(null, Array.from(deletedRecordsByTopic.values()));
-            });
-        });
-    }
-    #listOffsets(options, callback) {
-        this[kMetadata]({ topics: options.topics.map(topic => topic.name) }, (error, metadata) => {
-            if (error) {
-                callback(error);
-                return;
-            }
-            // metadata must be defined at this point
-            const { topics, brokers } = metadata;
-            const requests = new Map();
-            for (const topic of options.topics) {
-                for (const partition of topic.partitions) {
-                    // topics.get(topic.name) must be defined as the metadata request was successful
-                    const topicData = topics.get(topic.name);
-                    const targetPartitionData = topicData.partitions[partition.partitionIndex];
-                    /* c8 ignore next 4 - Hard to test */
-                    if (!targetPartitionData) {
-                        callback(new UserError(`Unknown partition ${partition.partitionIndex} for topic ${topic.name}.`));
-                        return;
-                    }
-                    let leaderRequests = requests.get(targetPartitionData.leader);
-                    if (!leaderRequests) {
-                        leaderRequests = [];
-                        requests.set(targetPartitionData.leader, leaderRequests);
-                    }
-                    let topicRequest = leaderRequests.find(t => t.name === topic.name);
-                    if (!topicRequest) {
-                        topicRequest = { name: topic.name, partitions: [] };
-                        leaderRequests.push(topicRequest);
-                    }
-                    topicRequest.partitions.push({
-                        partitionIndex: partition.partitionIndex,
-                        currentLeaderEpoch: targetPartitionData.leaderEpoch,
-                        /* c8 ignore next - Hard to test */
-                        timestamp: partition.timestamp ?? -1n
-                    });
-                }
-            }
-            runConcurrentCallbacks('Listing offsets failed.', requests, ([leader, requests], concurrentCallback) => {
-                this[kGetConnection](brokers.get(leader), (error, connection) => {
-                    if (error) {
-                        concurrentCallback(error);
-                        return;
-                    }
+                runConcurrentCallbacks('Listing offsets failed.', requests, ([leader, requests], concurrentCallback) => {
                     this[kPerformWithRetry]('listOffsets', retryCallback => {
-                        this[kGetApi]('ListOffsets', (error, api) => {
+                        this[kGetConnection](leader, (error, connection) => {
                             if (error) {
                                 retryCallback(error);
                                 return;
                             }
-                            api(connection, -1, options.isolationLevel ?? FetchIsolationLevels.READ_UNCOMMITTED, Array.from(requests.values()), retryCallback);
+                            this[kGetApi]('ListOffsets', (error, api) => {
+                                if (error) {
+                                    retryCallback(error);
+                                    return;
+                                }
+                                api(connection, -1, options.isolationLevel ?? FetchIsolationLevels.READ_UNCOMMITTED, requests, retryCallback);
+                            });
                         });
                     }, concurrentCallback, 0);
-                });
-            }, (error, responses) => {
-                if (error) {
-                    callback(error);
-                    return;
-                }
-                const ret = [];
-                for (const response of responses) {
-                    for (const topic of response.topics) {
-                        let topicOffsets = ret.find(t => t.name === topic.name);
-                        if (!topicOffsets) {
-                            topicOffsets = { name: topic.name, partitions: [] };
-                            ret.push(topicOffsets);
-                        }
-                        for (const partition of topic.partitions) {
-                            topicOffsets.partitions.push({
-                                offset: partition.offset,
-                                timestamp: partition.timestamp,
-                                partitionIndex: partition.partitionIndex,
-                                leaderEpoch: partition.leaderEpoch
-                            });
+                }, (error, responses) => {
+                    if (error) {
+                        retryCallback(error);
+                        return;
+                    }
+                    const ret = [];
+                    for (const response of responses) {
+                        for (const topic of response.topics) {
+                            let topicOffsets = ret.find(t => t.name === topic.name);
+                            if (!topicOffsets) {
+                                topicOffsets = { name: topic.name, partitions: [] };
+                                ret.push(topicOffsets);
+                            }
+                            for (const partition of topic.partitions) {
+                                topicOffsets.partitions.push({
+                                    offset: partition.offset,
+                                    timestamp: partition.timestamp,
+                                    partitionIndex: partition.partitionIndex,
+                                    leaderEpoch: partition.leaderEpoch
+                                });
+                            }
                         }
                     }
-                }
-                callback(null, ret);
+                    retryCallback(null, ret);
+                });
             });
-        });
+        }, callback, 0);
     }
 }