@platformatic/kafka 1.27.0 → 1.29.0

package/README.md

@@ -18,6 +18,10 @@ A modern, high-performance, pure TypeScript/JavaScript type safe client for Apac
 
 Supported Kafka version are from **3.5.0** to **4.0.0** and equivalent, edges included.
 
+## Supported KIPs
+
+See the maintained list in [`docs/kips.md`](./docs/kips.md).
+
 ## Installation
 
 ```bash
@@ -51,6 +55,7 @@ await producer.send({
 })
 
 // Close the producer when done
+// If you created producer streams with producer.asStream(), either close them first or use producer.close(true)
 await producer.close()
 ```
 
@@ -172,15 +177,17 @@ const producer = new Producer({
 
 // Send messages with schema IDs
 await producer.send({
-  messages: [{
-    topic: 'users',
-    value: { id: 1, name: 'Alice' },
-    metadata: {
-      schemas: {
-        value: 100 // Schema ID in the registry
+  messages: [
+    {
+      topic: 'users',
+      value: { id: 1, name: 'Alice' },
+      metadata: {
+        schemas: {
+          value: 100 // Schema ID in the registry
+        }
       }
     }
-  }]
+  ]
 })
 
 // Consumer with schema registry

package/dist/apis/admin/describe-cluster-v0.d.ts

@@ -18,6 +18,6 @@ export interface DescribeClusterResponse {
     brokers: DescribeClusterResponseBroker[];
     clusterAuthorizedOperations: number;
 }
-export declare function createRequest(includeClusterAuthorizedOperations: boolean): Writer;
+export declare function createRequest(includeClusterAuthorizedOperations: boolean, _endpointType: number): Writer;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): DescribeClusterResponse;
-export declare const api: import("../definitions.ts").API<[includeClusterAuthorizedOperations: boolean], DescribeClusterResponse>;
+export declare const api: import("../definitions.ts").API<[includeClusterAuthorizedOperations: boolean, _endpointType: number], DescribeClusterResponse>;

package/dist/apis/admin/describe-cluster-v0.js

@@ -5,7 +5,7 @@ import { createAPI } from "../definitions.js";
   DescribeCluster Request (Version: 0) => include_cluster_authorized_operations TAG_BUFFER
     include_cluster_authorized_operations => BOOLEAN
 */
-export function createRequest(includeClusterAuthorizedOperations) {
+export function createRequest(includeClusterAuthorizedOperations, _endpointType) {
     return Writer.create().appendBoolean(includeClusterAuthorizedOperations).appendTaggedFields();
 }
 /*

package/dist/apis/admin/list-groups-v4.d.ts

@@ -12,6 +12,6 @@ export interface ListGroupsResponse {
     errorCode: number;
     groups: ListGroupsResponseGroup[];
 }
-export declare function createRequest(statesFilter: ConsumerGroupStateValue[]): Writer;
+export declare function createRequest(statesFilter: ConsumerGroupStateValue[], _typesFilter: string[]): Writer;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ListGroupsResponse;
-export declare const api: import("../definitions.ts").API<[statesFilter: ("PREPARING_REBALANCE" | "COMPLETING_REBALANCE" | "STABLE" | "DEAD" | "EMPTY")[]], ListGroupsResponse>;
+export declare const api: import("../definitions.ts").API<[statesFilter: ("PREPARING_REBALANCE" | "COMPLETING_REBALANCE" | "STABLE" | "DEAD" | "EMPTY")[], _typesFilter: string[]], ListGroupsResponse>;

package/dist/apis/admin/list-groups-v4.js

@@ -6,7 +6,7 @@ import { createAPI } from "../definitions.js";
   ListGroups Request (Version: 4) => [states_filter] TAG_BUFFER
     states_filter => COMPACT_STRING
 */
-export function createRequest(statesFilter) {
+export function createRequest(statesFilter, _typesFilter) {
     return Writer.create()
         .appendArray(statesFilter, (w, s) => w.appendString(pascalCase(s, { normalize: true })), true, false)
         .appendTaggedFields();

package/dist/apis/admin/list-transactions-v0.d.ts

@@ -13,6 +13,6 @@ export interface ListTransactionsResponse {
     unknownStateFilters: string[];
     transactionStates: ListTransactionsResponseTransactionState[];
 }
-export declare function createRequest(stateFilters: TransactionState[], producerIdFilters: bigint[]): Writer;
+export declare function createRequest(stateFilters: TransactionState[], producerIdFilters: bigint[], _durationFilter: bigint): Writer;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ListTransactionsResponse;
-export declare const api: import("../definitions.ts").API<[stateFilters: ("EMPTY" | "ONGOING" | "PREPARE_ABORT" | "COMMITTING" | "ABORTING" | "COMPLETE_COMMIT" | "COMPLETE_ABORT")[], producerIdFilters: bigint[]], ListTransactionsResponse>;
+export declare const api: import("../definitions.ts").API<[stateFilters: ("EMPTY" | "ONGOING" | "PREPARE_ABORT" | "COMMITTING" | "ABORTING" | "COMPLETE_COMMIT" | "COMPLETE_ABORT")[], producerIdFilters: bigint[], _durationFilter: bigint], ListTransactionsResponse>;

package/dist/apis/admin/list-transactions-v0.js

@@ -6,7 +6,7 @@ import { createAPI } from "../definitions.js";
     state_filters => COMPACT_STRING
     producer_id_filters => INT64
 */
-export function createRequest(stateFilters, producerIdFilters) {
+export function createRequest(stateFilters, producerIdFilters, _durationFilter) {
     return Writer.create()
         .appendArray(stateFilters, (w, t) => w.appendString(t), true, false)
         .appendArray(producerIdFilters, (w, p) => w.appendInt64(p), true, false)

package/dist/apis/consumer/join-group-v6.d.ts

@@ -22,6 +22,6 @@ export interface JoinGroupResponse {
     memberId: NullableString;
     members: JoinGroupResponseMember[];
 }
-export declare function createRequest(groupId: string, sessionTimeoutMs: number, rebalanceTimeoutMs: number, memberId: string, groupInstanceId: NullableString, protocolType: string, protocols: JoinGroupRequestProtocol[]): Writer;
+export declare function createRequest(groupId: string, sessionTimeoutMs: number, rebalanceTimeoutMs: number, memberId: string, groupInstanceId: NullableString, protocolType: string, protocols: JoinGroupRequestProtocol[], _reason?: NullableString): Writer;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): JoinGroupResponse;
-export declare const api: import("../definitions.ts").API<[groupId: string, sessionTimeoutMs: number, rebalanceTimeoutMs: number, memberId: string, groupInstanceId: NullableString, protocolType: string, protocols: JoinGroupRequestProtocol[]], JoinGroupResponse>;
+export declare const api: import("../definitions.ts").API<[groupId: string, sessionTimeoutMs: number, rebalanceTimeoutMs: number, memberId: string, groupInstanceId: NullableString, protocolType: string, protocols: JoinGroupRequestProtocol[], _reason?: NullableString], JoinGroupResponse>;

package/dist/apis/consumer/join-group-v6.js

@@ -14,7 +14,7 @@ import { createAPI } from "../definitions.js";
       name => COMPACT_STRING
       metadata => COMPACT_BYTES
 */
-export function createRequest(groupId, sessionTimeoutMs, rebalanceTimeoutMs, memberId, groupInstanceId, protocolType, protocols) {
+export function createRequest(groupId, sessionTimeoutMs, rebalanceTimeoutMs, memberId, groupInstanceId, protocolType, protocols, _reason) {
     return Writer.create()
         .appendString(groupId)
         .appendInt32(sessionTimeoutMs)

package/dist/apis/consumer/join-group-v7.d.ts

@@ -22,6 +22,6 @@ export interface JoinGroupResponse {
     memberId: NullableString;
     members: JoinGroupResponseMember[];
 }
-export declare function createRequest(groupId: string, sessionTimeoutMs: number, rebalanceTimeoutMs: number, memberId: string, groupInstanceId: NullableString, protocolType: string, protocols: JoinGroupRequestProtocol[]): Writer;
+export declare function createRequest(groupId: string, sessionTimeoutMs: number, rebalanceTimeoutMs: number, memberId: string, groupInstanceId: NullableString, protocolType: string, protocols: JoinGroupRequestProtocol[], _reason?: NullableString): Writer;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): JoinGroupResponse;
-export declare const api: import("../definitions.ts").API<[groupId: string, sessionTimeoutMs: number, rebalanceTimeoutMs: number, memberId: string, groupInstanceId: NullableString, protocolType: string, protocols: JoinGroupRequestProtocol[]], JoinGroupResponse>;
+export declare const api: import("../definitions.ts").API<[groupId: string, sessionTimeoutMs: number, rebalanceTimeoutMs: number, memberId: string, groupInstanceId: NullableString, protocolType: string, protocols: JoinGroupRequestProtocol[], _reason?: NullableString], JoinGroupResponse>;

package/dist/apis/consumer/join-group-v7.js

@@ -14,7 +14,7 @@ import { createAPI } from "../definitions.js";
       name => COMPACT_STRING
       metadata => COMPACT_BYTES
 */
-export function createRequest(groupId, sessionTimeoutMs, rebalanceTimeoutMs, memberId, groupInstanceId, protocolType, protocols) {
+export function createRequest(groupId, sessionTimeoutMs, rebalanceTimeoutMs, memberId, groupInstanceId, protocolType, protocols, _reason) {
     return Writer.create()
         .appendString(groupId)
         .appendInt32(sessionTimeoutMs)

package/dist/apis/consumer/offset-commit-v8.d.ts

@@ -24,6 +24,6 @@ export interface OffsetCommitResponse {
     throttleTimeMs: number;
     topics: OffsetCommitResponseTopic[];
 }
-export declare function createRequest(groupId: string, generationIdOrMemberEpoch: number, memberId: string, groupInstanceId: NullableString, topics: OffsetCommitRequestTopic[]): Writer;
+export declare function createRequest(groupId: string, generationIdOrMemberEpoch: number, memberId: NullableString, groupInstanceId: NullableString, topics: OffsetCommitRequestTopic[]): Writer;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): OffsetCommitResponse;
-export declare const api: import("../definitions.ts").API<[groupId: string, generationIdOrMemberEpoch: number, memberId: string, groupInstanceId: NullableString, topics: OffsetCommitRequestTopic[]], OffsetCommitResponse>;
+export declare const api: import("../definitions.ts").API<[groupId: string, generationIdOrMemberEpoch: number, memberId: NullableString, groupInstanceId: NullableString, topics: OffsetCommitRequestTopic[]], OffsetCommitResponse>;

package/dist/apis/consumer/sync-group-v4.d.ts

@@ -13,6 +13,6 @@ export interface SyncGroupResponse {
     protocolName: NullableString;
     assignment: Buffer;
 }
-export declare function createRequest(groupId: string, generationId: number, memberId: string, groupInstanceId: NullableString, assignments: SyncGroupRequestAssignment[]): Writer;
+export declare function createRequest(groupId: string, generationId: number, memberId: string, groupInstanceId: NullableString, _protocolType: NullableString, _protocolName: NullableString, assignments: SyncGroupRequestAssignment[]): Writer;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): SyncGroupResponse;
-export declare const api: import("../definitions.ts").API<[groupId: string, generationId: number, memberId: string, groupInstanceId: NullableString, assignments: SyncGroupRequestAssignment[]], SyncGroupResponse>;
+export declare const api: import("../definitions.ts").API<[groupId: string, generationId: number, memberId: string, groupInstanceId: NullableString, _protocolType: NullableString, _protocolName: NullableString, assignments: SyncGroupRequestAssignment[]], SyncGroupResponse>;

package/dist/apis/consumer/sync-group-v4.js

@@ -12,7 +12,7 @@ import { createAPI } from "../definitions.js";
       assignment => COMPACT_BYTES
 
 */
-export function createRequest(groupId, generationId, memberId, groupInstanceId, assignments) {
+export function createRequest(groupId, generationId, memberId, groupInstanceId, _protocolType, _protocolName, assignments) {
     return Writer.create()
         .appendString(groupId)
         .appendInt32(generationId)

package/dist/clients/admin/admin.d.ts

@@ -5,7 +5,7 @@ import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type Callback } from '../../apis/definitions.ts';
 import { type Acl } from '../../apis/types.ts';
 import { Base } from '../base/base.ts';
-import { type AdminListOffsetsOptions, type AdminOptions, type AlterClientQuotasOptions, type AlterConfigsOptions, type AlterConsumerGroupOffsetsOptions, type BrokerLogDirDescription, type ConfigDescription, type CreateAclsOptions, type CreatedTopic, type CreatePartitionsOptions, type CreateTopicsOptions, type DeleteAclsOptions, type DeleteConsumerGroupOffsetsOptions, type DeleteGroupsOptions, type DeleteTopicsOptions, type DescribeAclsOptions, type DescribeClientQuotasOptions, type DescribeConfigsOptions, type DescribeGroupsOptions, type DescribeLogDirsOptions, type Group, type GroupBase, type IncrementalAlterConfigsOptions, type ListConsumerGroupOffsetsGroup, type ListConsumerGroupOffsetsOptions, type ListedOffsetsTopic, type ListGroupsOptions, type ListTopicsOptions, type RemoveMembersFromConsumerGroupOptions } from './types.ts';
+import { type AdminListOffsetsOptions, type AdminOptions, type AlterClientQuotasOptions, type AlterConfigsOptions, type AlterConsumerGroupOffsetsOptions, type BrokerLogDirDescription, type ConfigDescription, type CreateAclsOptions, type CreatedTopic, type CreatePartitionsOptions, type CreateTopicsOptions, type DeleteAclsOptions, type DeleteConsumerGroupOffsetsOptions, type DeleteRecordsOptions, type DeleteGroupsOptions, type DeleteTopicsOptions, type DeletedRecordsTopic, type DescribeAclsOptions, type DescribeClientQuotasOptions, type DescribeConfigsOptions, type DescribeGroupsOptions, type DescribeLogDirsOptions, type Group, type GroupBase, type IncrementalAlterConfigsOptions, type ListConsumerGroupOffsetsGroup, type ListConsumerGroupOffsetsOptions, type ListedOffsetsTopic, type ListGroupsOptions, type ListTopicsOptions, type RemoveMembersFromConsumerGroupOptions } from './types.ts';
 export declare class Admin extends Base<AdminOptions> {
     #private;
     constructor(options: AdminOptions);
@@ -57,4 +57,6 @@ export declare class Admin extends Base<AdminOptions> {
     deleteAcls(options: DeleteAclsOptions): Promise<Acl[]>;
     listOffsets(options: AdminListOffsetsOptions, callback: CallbackWithPromise<ListedOffsetsTopic[]>): void;
     listOffsets(options: AdminListOffsetsOptions): Promise<ListedOffsetsTopic[]>;
+    deleteRecords(options: DeleteRecordsOptions, callback: CallbackWithPromise<DeletedRecordsTopic[]>): void;
+    deleteRecords(options: DeleteRecordsOptions): Promise<DeletedRecordsTopic[]>;
 }

package/dist/clients/admin/admin.js

@@ -4,7 +4,7 @@ import { adminAclsChannel, adminClientQuotasChannel, adminConfigsChannel, adminC
 import { MultipleErrors } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Base, kAfterCreate, kCheckNotClosed, kConnections, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
-import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteGroupsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
+import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteRecordsOptionsValidator, deleteGroupsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
 export class Admin extends Base {
     #controller = null;
     constructor(options) {
@@ -334,6 +334,21 @@ export class Admin extends Base {
         adminOffsetsChannel.traceCallback(this.#listOffsets, 1, createDiagnosticContext({ client: this, operation: 'listOffsets', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
+    deleteRecords(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        const validationError = this[kValidateOptions](options, deleteRecordsOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError);
+            return callback[kCallbackPromise];
+        }
+        adminOffsetsChannel.traceCallback(this.#deleteRecords, 1, createDiagnosticContext({ client: this, operation: 'deleteRecords', options }), this, options, callback);
+        return callback[kCallbackPromise];
+    }
     #getControllerConnection(callback) {
         if (this.#controller) {
             this[kConnections].get(this.#controller, callback);
@@ -518,12 +533,7 @@ export class Admin extends Base {
                                 return;
                             }
                             /* c8 ignore next 5 */
-                            if (api.version === 4) {
-                                api(connection, options.states ?? [], retryCallback);
-                            }
-                            else {
-                                api(connection, options.states ?? [], options.types, retryCallback);
-                            }
+                            api(connection, options.states ?? [], options.types, retryCallback);
                         });
                     }, concurrentCallback, 0);
                 });
@@ -1276,6 +1286,77 @@ export class Admin extends Base {
             })));
         }, 0);
     }
+    #deleteRecords(options, callback) {
+        this[kMetadata]({ topics: options.topics.map(topic => topic.name) }, (error, metadata) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            const requestsByLeader = new Map();
+            for (const topic of options.topics) {
+                for (const partition of topic.partitions) {
+                    const { leader } = metadata.topics.get(topic.name).partitions[partition.partition];
+                    let leaderRequests = requestsByLeader.get(leader);
+                    if (!leaderRequests) {
+                        leaderRequests = new Map();
+                        requestsByLeader.set(leader, leaderRequests);
+                    }
+                    let topicRequest = leaderRequests.get(topic.name);
+                    if (!topicRequest) {
+                        topicRequest = { name: topic.name, partitions: [] };
+                        leaderRequests.set(topic.name, topicRequest);
+                    }
+                    topicRequest.partitions.push({
+                        partitionIndex: partition.partition,
+                        offset: partition.offset
+                    });
+                }
+            }
+            const requests = new Map();
+            for (const [leader, topics] of requestsByLeader) {
+                requests.set(leader, Array.from(topics.values()));
+            }
+            runConcurrentCallbacks('Deleting records failed.', requests, ([leader, requests], concurrentCallback) => {
+                this[kGetConnection](metadata.brokers.get(leader), (error, connection) => {
+                    if (error) {
+                        concurrentCallback(error);
+                        return;
+                    }
+                    this[kPerformWithRetry]('deleteRecords', retryCallback => {
+                        this[kGetApi]('DeleteRecords', (error, api) => {
+                            if (error) {
+                                retryCallback(error);
+                                return;
+                            }
+                            api(connection, Array.from(requests.values()), this[kOptions].timeout, retryCallback);
+                        });
+                    }, concurrentCallback, 0);
+                });
+            }, (error, responses) => {
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                const deletedRecordsByTopic = new Map();
+                for (const response of responses) {
+                    for (const topic of response.topics) {
+                        let topicDeletedRecords = deletedRecordsByTopic.get(topic.name);
+                        if (!topicDeletedRecords) {
+                            topicDeletedRecords = { name: topic.name, partitions: [] };
+                            deletedRecordsByTopic.set(topic.name, topicDeletedRecords);
+                        }
+                        for (const partition of topic.partitions) {
+                            topicDeletedRecords.partitions.push({
+                                partition: partition.partitionIndex,
+                                lowWatermark: partition.lowWatermark
+                            });
+                        }
+                    }
+                }
+                callback(null, Array.from(deletedRecordsByTopic.values()));
+            });
+        });
+    }
     #listOffsets(options, callback) {
         this[kMetadata]({ topics: options.topics.map(topic => topic.name) }, (error, metadata) => {
             if (error) {

package/dist/clients/admin/options.d.ts

@@ -484,6 +484,46 @@ export declare const deleteConsumerGroupOffsetsOptionsSchema: {
     required: string[];
     additionalProperties: boolean;
 };
+export declare const deleteRecordsOptionsSchema: {
+    type: string;
+    properties: {
+        topics: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    name: {
+                        type: string;
+                        minLength: number;
+                    };
+                    partitions: {
+                        type: string;
+                        items: {
+                            type: string;
+                            properties: {
+                                partition: {
+                                    type: string;
+                                    minimum: number;
+                                };
+                                offset: {
+                                    bigint: boolean;
+                                };
+                            };
+                            required: string[];
+                            additionalProperties: boolean;
+                        };
+                        minItems: number;
+                    };
+                };
+                required: string[];
+                additionalProperties: boolean;
+            };
+            minItems: number;
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
 export declare const describeConfigsOptionsSchema: {
     type: string;
     properties: {
@@ -881,6 +921,9 @@ export declare const alterConsumerGroupOffsetsOptionsValidator: import("ajv").Va
 export declare const deleteConsumerGroupOffsetsOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;
+export declare const deleteRecordsOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
 export declare const listConsumerGroupOffsetsOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;

package/dist/clients/admin/options.js

@@ -343,6 +343,38 @@ export const deleteConsumerGroupOffsetsOptionsSchema = {
     required: ['groupId', 'topics'],
     additionalProperties: false
 };
+export const deleteRecordsOptionsSchema = {
+    type: 'object',
+    properties: {
+        topics: {
+            type: 'array',
+            items: {
+                type: 'object',
+                properties: {
+                    name: { type: 'string', minLength: 1 },
+                    partitions: {
+                        type: 'array',
+                        items: {
+                            type: 'object',
+                            properties: {
+                                partition: { type: 'number', minimum: 0 },
+                                offset: { bigint: true }
+                            },
+                            required: ['partition', 'offset'],
+                            additionalProperties: false
+                        },
+                        minItems: 1
+                    }
+                },
+                required: ['name', 'partitions'],
+                additionalProperties: false
+            },
+            minItems: 1
+        }
+    },
+    required: ['topics'],
+    additionalProperties: false
+};
 export const describeConfigsOptionsSchema = {
     type: 'object',
     properties: {
@@ -568,6 +600,7 @@ export const alterClientQuotasOptionsValidator = ajv.compile(alterClientQuotasOp
 export const describeLogDirsOptionsValidator = ajv.compile(describeLogDirsOptionsSchema);
 export const alterConsumerGroupOffsetsOptionsValidator = ajv.compile(alterConsumerGroupOffsetsOptionsSchema);
 export const deleteConsumerGroupOffsetsOptionsValidator = ajv.compile(deleteConsumerGroupOffsetsOptionsSchema);
+export const deleteRecordsOptionsValidator = ajv.compile(deleteRecordsOptionsSchema);
 export const listConsumerGroupOffsetsOptionsValidator = ajv.compile(listConsumerGroupOffsetsOptionsSchema);
 export const describeConfigsOptionsValidator = ajv.compile(describeConfigsOptionsSchema);
 export const alterConfigsOptionsValidator = ajv.compile(alterConfigsOptionsSchema);

package/dist/clients/admin/types.d.ts

@@ -139,6 +139,25 @@ export interface DeleteConsumerGroupOffsetsOptions {
         partitionIndexes: number[];
     }[];
 }
+export interface DeleteRecordsPartitionOffset {
+    partition: number;
+    offset: bigint;
+}
+export interface DeleteRecordsTopic {
+    name: string;
+    partitions: DeleteRecordsPartitionOffset[];
+}
+export interface DeleteRecordsOptions {
+    topics: DeleteRecordsTopic[];
+}
+export interface DeletedRecordsPartition {
+    partition: number;
+    lowWatermark: bigint;
+}
+export interface DeletedRecordsTopic {
+    name: string;
+    partitions: DeletedRecordsPartition[];
+}
 export interface DescribeConfigsOptions {
     resources: DescribeConfigsRequestResource[];
     includeSynonyms?: boolean;

package/dist/clients/base/base.js

@@ -312,7 +312,12 @@ export class Base extends TypedEventEmitter {
         this[kConnections].get(broker, callback);
     }
     [kGetBootstrapConnection](callback) {
-        this[kConnections].getFirstAvailable(this[kBootstrapBrokers], callback);
+        if (!this.#metadata) {
+            this[kConnections].getFirstAvailable(this[kBootstrapBrokers], callback);
+            return;
+        }
+        const discovered = Array.from(this.#metadata.brokers.values());
+        this[kConnections].getFirstAvailable([...this[kBootstrapBrokers], ...discovered], callback);
     }
     [kValidateOptions](target, validator, targetName, throwOnErrors = true) {
         if (!this[kOptions].strict) {

package/dist/clients/consumer/messages-stream.js

@@ -37,6 +37,9 @@ export class MessagesStream extends Readable {
     #mode;
     #fallbackMode;
     #paused;
+    #refreshOffsetsInflight;
+    #refreshOffsetsPending;
+    #refreshOffsetsDestroyOnError;
     #fetches;
     #maxFetches;
     #options;
@@ -99,6 +102,9 @@ export class MessagesStream extends Readable {
         this.#offsetsCommitted = new Map();
         this.#partitionsEpochs = new Map();
         this.#paused = false;
+        this.#refreshOffsetsInflight = false;
+        this.#refreshOffsetsPending = false;
+        this.#refreshOffsetsDestroyOnError = false;
         this.#fetches = 0;
         this.#maxFetches = maxFetches ?? 0;
         this.#topics = structuredClone(options.topics);
@@ -144,14 +150,7 @@ export class MessagesStream extends Readable {
         // having some.
         this.#consumer.on('consumer:group:join', () => {
             this.#offsetsCommitted.clear();
-            this.#refreshOffsets(error => {
-                /* c8 ignore next 4 - Hard to test */
-                if (error) {
-                    this.destroy(error);
-                    return;
-                }
-                this.#fetch();
-            });
+            this.#scheduleRefreshOffsetsAndFetch();
         });
         if (consumer[kPrometheus]) {
             this.#metricsConsumedMessages = ensureMetric(consumer[kPrometheus], 'Counter', 'kafka_consumed_messages', 'Number of consumed Kafka messages');
@@ -291,7 +290,7 @@ export class MessagesStream extends Readable {
             return;
         }
         // No need to fetch if nobody is consuming the data
-        if (this.readableFlowing === null || this.#paused) {
+        if (this.readableFlowing === null || this.#paused || this.#refreshOffsetsInflight || this.#refreshOffsetsPending) {
             return;
         }
         this.#consumer.metadata({ topics: this.#consumer.topics.current }, (error, metadata) => {
@@ -448,8 +447,11 @@ export class MessagesStream extends Readable {
                         try {
                             const headers = new Map();
                             for (const [headerKey, headerValue] of record.headers) {
-                                headers.set(headerKeyDeserializer(headerKey ?? undefined, messageToConsume), headerValueDeserializer(headerValue ?? undefined, messageToConsume));
+                                headers.set(
+                                /* c8 ignore next 2 - Hard to test */
+                                headerKeyDeserializer(headerKey ?? undefined, messageToConsume), headerValueDeserializer(headerValue ?? undefined, messageToConsume));
                             }
+                            /* c8 ignore next 2 - Hard to test */
                             const key = keyDeserializer(record.key ?? undefined, headers, messageToConsume);
                             const value = valueDeserializer(record.value ?? undefined, headers, messageToConsume);
                             this.#metricsConsumedMessages?.inc();
@@ -595,11 +597,38 @@ export class MessagesStream extends Readable {
             });
         });
     }
-    [kRefreshOffsetsAndFetch]() {
-        this.#refreshOffsets(() => {
+    #scheduleRefreshOffsetsAndFetch(destroyOnError = true) {
+        this.#refreshOffsetsDestroyOnError ||= destroyOnError;
+        /* c8 ignore next 4 - Hard to test */
+        if (this.#refreshOffsetsInflight) {
+            this.#refreshOffsetsPending = true;
+            return;
+        }
+        this.#refreshOffsetsInflight = true;
+        this.#refreshOffsets(error => {
+            const shouldDestroyOnError = this.#refreshOffsetsDestroyOnError;
+            this.#refreshOffsetsInflight = false;
+            this.#refreshOffsetsDestroyOnError = false;
+            this.#refreshOffsetsPending = false;
+            /* c8 ignore next 9 - Hard to test */
+            if (error) {
+                if (shouldDestroyOnError) {
+                    this.destroy(error);
+                }
+                return;
+            }
+            // A new one was scheduled while the previous one was inflight, we need to run it immediately
+            /* c8 ignore next 4 - Hard to test */
+            if (this.#refreshOffsetsPending) {
+                this.#scheduleRefreshOffsetsAndFetch(shouldDestroyOnError);
+                return;
+            }
             this.#fetch();
         });
     }
+    [kRefreshOffsetsAndFetch]() {
+        this.#scheduleRefreshOffsetsAndFetch(false);
+    }
     #assignOffsets(offsets, commits, callback) {
         for (const [topic, partitions] of offsets) {
             for (let i = 0; i < partitions.length; i++) {

package/dist/clients/producer/index.d.ts

@@ -1,3 +1,4 @@
 export * from './options.ts';
+export * from './producer-stream.ts';
 export * from './producer.ts';
 export * from './types.ts';

package/dist/clients/producer/index.js

@@ -1,3 +1,4 @@
 export * from "./options.js";
+export * from "./producer-stream.js";
 export * from "./producer.js";
 export * from "./types.js";

package/dist/clients/producer/options.d.ts

@@ -155,3 +155,66 @@ export declare const sendOptionsSchema: {
 export declare const sendOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;
+export declare const defaultProducerStreamOptions: {
+    batchSize: number;
+    batchTime: number;
+    reportMode: "none";
+};
+export declare const producerStreamOptionsSchema: {
+    type: string;
+    properties: {
+        producerId: {
+            bigint: boolean;
+        };
+        producerEpoch: {
+            type: string;
+        };
+        idempotent: {
+            type: string;
+        };
+        transactionalId: {
+            type: string;
+        };
+        acks: {
+            type: string;
+            enumeration: {
+                allowed: (0 | 1 | -1)[];
+                errorMessage: string;
+            };
+        };
+        compression: {
+            type: string;
+            enumeration: {
+                allowed: import("../../protocol/compression.ts").CompressionAlgorithmValue[];
+                errorMessage: string;
+            };
+        };
+        partitioner: {
+            function: boolean;
+        };
+        autocreateTopics: {
+            type: string;
+        };
+        repeatOnStaleMetadata: {
+            type: string;
+        };
+        highWaterMark: {
+            type: string;
+            minimum: number;
+        };
+        batchSize: {
+            type: string;
+            minimum: number;
+        };
+        batchTime: {
+            type: string;
+            minimum: number;
+        };
+        reportMode: {
+            type: string;
+            enum: import("./types.ts").ProducerStreamReportModeValue[];
+        };
+    };
+    additionalProperties: boolean;
+};
+export declare const producerStreamOptionsValidator: import("ajv").ValidateFunction<unknown>;