@platformatic/kafka 1.6.0 → 1.8.0

package/README.md

@@ -224,6 +224,10 @@ await consumer.close()
 await producer.close()
 ```
 
+### Avro
+
+An example on how to use [Avro]() for (de)serialisation can be found in the [examples](./examples) folder.
+
 ### Error Handling
 
 `@platformatic/kafka` defines its hierarchy of errors.

package/dist/clients/admin/admin.d.ts

@@ -1,10 +1,12 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type Callback } from '../../apis/definitions.ts';
 import { Base } from '../base/base.ts';
-import { type AdminOptions, type CreatedTopic, type CreateTopicsOptions, type DeleteGroupsOptions, type DeleteTopicsOptions, type DescribeGroupsOptions, type Group, type GroupBase, type ListGroupsOptions } from './types.ts';
+import { type AdminOptions, type CreatedTopic, type CreateTopicsOptions, type DeleteGroupsOptions, type DeleteTopicsOptions, type DescribeGroupsOptions, type Group, type GroupBase, type ListGroupsOptions, type ListTopicsOptions } from './types.ts';
 export declare class Admin extends Base<AdminOptions> {
     #private;
     constructor(options: AdminOptions);
+    listTopics(options: ListTopicsOptions, callback: Callback<string[]>): void;
+    listTopics(options?: ListTopicsOptions): Promise<string[]>;
     createTopics(options: CreateTopicsOptions, callback: Callback<CreatedTopic[]>): void;
     createTopics(options: CreateTopicsOptions): Promise<CreatedTopic[]>;
     deleteTopics(options: DeleteTopicsOptions, callback: CallbackWithPromise<void>): void;

package/dist/clients/admin/admin.js

@@ -3,12 +3,30 @@ import { FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { adminGroupsChannel, adminTopicsChannel, createDiagnosticContext } from "../../diagnostic.js";
 import { Reader } from "../../protocol/reader.js";
 import { Base, kAfterCreate, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
-import { createTopicsOptionsValidator, deleteGroupsOptionsValidator, deleteTopicsOptionsValidator, describeGroupsOptionsValidator, listGroupsOptionsValidator } from "./options.js";
+import { createTopicsOptionsValidator, deleteGroupsOptionsValidator, deleteTopicsOptionsValidator, describeGroupsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator } from "./options.js";
 export class Admin extends Base {
     constructor(options) {
         super(options);
         this[kAfterCreate]('admin');
     }
+    listTopics(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        if (!options) {
+            options = {};
+        }
+        const validationError = this[kValidateOptions](options, listTopicsOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError, undefined);
+            return callback[kCallbackPromise];
+        }
+        adminTopicsChannel.traceCallback(this.#listTopics, 1, createDiagnosticContext({ client: this, operation: 'listTopics', options }), this, options, callback);
+        return callback[kCallbackPromise];
+    }
     createTopics(options, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -88,6 +106,40 @@ export class Admin extends Base {
         adminGroupsChannel.traceCallback(this.#deleteGroups, 1, createDiagnosticContext({ client: this, operation: 'deleteGroups', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
+    #listTopics(options, callback) {
+        const includeInternals = options.includeInternals ?? false;
+        this[kPerformDeduplicated]('metadata', deduplicateCallback => {
+            this[kPerformWithRetry]('metadata', retryCallback => {
+                this[kGetBootstrapConnection]((error, connection) => {
+                    if (error) {
+                        retryCallback(error, undefined);
+                        return;
+                    }
+                    this[kGetApi]('Metadata', (error, api) => {
+                        if (error) {
+                            retryCallback(error, undefined);
+                            return;
+                        }
+                        api(connection, null, false, false, retryCallback);
+                    });
+                });
+            }, (error, metadata) => {
+                if (error) {
+                    deduplicateCallback(error, undefined);
+                    return;
+                }
+                const topics = new Set();
+                for (const { name, isInternal } of metadata.topics) {
+                    /* c8 ignore next 3 - Sometimes internal topics might be returned by Kafka */
+                    if (isInternal && !includeInternals) {
+                        continue;
+                    }
+                    topics.add(name);
+                }
+                deduplicateCallback(null, Array.from(topics).sort());
+            }, 0);
+        }, callback);
+    }
     #createTopics(options, callback) {
         const numPartitions = options.partitions ?? 1;
         const replicationFactor = options.replicas ?? 1;

package/dist/clients/admin/options.d.ts

@@ -50,6 +50,16 @@ export declare const createTopicOptionsSchema: {
     required: string[];
     additionalProperties: boolean;
 };
+export declare const listTopicOptionsSchema: {
+    type: string;
+    properties: {
+        includeInternals: {
+            type: string;
+            default: boolean;
+        };
+    };
+    additionalProperties: boolean;
+};
 export declare const deleteTopicOptionsSchema: {
     type: string;
     properties: {
@@ -71,8 +81,10 @@ export declare const listGroupsOptionsSchema: {
             type: string;
             items: {
                 type: string;
-                enum: readonly ["PREPARING_REBALANCE", "COMPLETING_REBALANCE", "STABLE", "DEAD", "EMPTY"];
-                errorMessage: string;
+                enumeration: {
+                    allowed: readonly ["PREPARING_REBALANCE", "COMPLETING_REBALANCE", "STABLE", "DEAD", "EMPTY"];
+                    errorMessage: string;
+                };
             };
             minItems: number;
         };
@@ -123,6 +135,7 @@ export declare const deleteGroupsOptionsSchema: {
 export declare const createTopicsOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;
+export declare const listTopicsOptionsValidator: import("ajv").ValidateFunction<unknown>;
 export declare const deleteTopicsOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;

package/dist/clients/admin/options.js

@@ -31,6 +31,13 @@ export const createTopicOptionsSchema = {
     required: ['topics'],
     additionalProperties: false
 };
+export const listTopicOptionsSchema = {
+    type: 'object',
+    properties: {
+        includeInternals: { type: 'boolean', default: false }
+    },
+    additionalProperties: false
+};
 export const deleteTopicOptionsSchema = {
     type: 'object',
     properties: {
@@ -46,8 +53,10 @@ export const listGroupsOptionsSchema = {
             type: 'array',
             items: {
                 type: 'string',
-                enum: ConsumerGroupStates,
-                errorMessage: listErrorMessage(ConsumerGroupStates)
+                enumeration: {
+                    allowed: ConsumerGroupStates,
+                    errorMessage: listErrorMessage(ConsumerGroupStates)
+                }
             },
             minItems: 0
         },
@@ -75,6 +84,7 @@ export const deleteGroupsOptionsSchema = {
     additionalProperties: false
 };
 export const createTopicsOptionsValidator = ajv.compile(createTopicOptionsSchema);
+export const listTopicsOptionsValidator = ajv.compile(listTopicOptionsSchema);
 export const deleteTopicsOptionsValidator = ajv.compile(deleteTopicOptionsSchema);
 export const listGroupsOptionsValidator = ajv.compile(listGroupsOptionsSchema);
 export const describeGroupsOptionsValidator = ajv.compile(describeGroupsOptionsSchema);

package/dist/clients/admin/types.d.ts

@@ -40,6 +40,9 @@ export interface CreateTopicsOptions {
     replicas?: number;
     assignments?: BrokerAssignment[];
 }
+export interface ListTopicsOptions {
+    includeInternals?: boolean;
+}
 export interface DeleteTopicsOptions {
     topics: string[];
 }

package/dist/clients/base/base.js

@@ -148,13 +148,26 @@ export class Base extends EventEmitter {
         }, callback);
     }
     [kMetadata](options, callback) {
-        const metadataMaxAge = options.metadataMaxAge ?? this[kOptions].metadataMaxAge;
-        const isStale = options.forceUpdate ||
-            !this.#metadata ||
-            Date.now() > this.#metadata.lastUpdate + metadataMaxAge ||
-            options.topics.some(topic => !this.#metadata?.topics.has(topic));
-        if (!isStale) {
-            callback(null, this.#metadata);
+        const expiralDate = Date.now() - (options.metadataMaxAge ?? this[kOptions].metadataMaxAge);
+        let topicsToFetch = [];
+        // Determine which topics we need to fetch
+        if (!this.#metadata || options.forceUpdate) {
+            topicsToFetch = options.topics;
+        }
+        else {
+            for (const topic of options.topics) {
+                const existingTopic = this.#metadata.topics.get(topic);
+                if (!existingTopic || existingTopic.lastUpdate < expiralDate) {
+                    topicsToFetch.push(topic);
+                }
+            }
+        }
+        // All topics are already up-to-date, simply return them
+        if (this.#metadata && !topicsToFetch.length) {
+            callback(null, {
+                ...this.#metadata,
+                topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
+            });
             return;
         }
         const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
@@ -178,12 +191,26 @@ export class Base extends EventEmitter {
                     deduplicateCallback(error, undefined);
                     return;
                 }
+                const lastUpdate = Date.now();
+                if (!this.#metadata) {
+                    this.#metadata = {
+                        id: metadata.clusterId,
+                        brokers: new Map(),
+                        topics: new Map(),
+                        lastUpdate
+                    };
+                }
+                else {
+                    this.#metadata.lastUpdate = lastUpdate;
+                }
                 const brokers = new Map();
-                const topics = new Map();
+                // This should never change, but we act defensively here
                 for (const broker of metadata.brokers) {
                     const { host, port } = broker;
                     brokers.set(broker.nodeId, { host, port });
                 }
+                this.#metadata.brokers = brokers;
+                // Update all the topics in the cache
                 for (const { name, topicId: id, partitions: rawPartitions, isInternal } of metadata.topics) {
                     /* c8 ignore next 3 - Sometimes internal topics might be returned by Kafka */
                     if (isInternal) {
@@ -197,16 +224,15 @@ export class Base extends EventEmitter {
                             replicas: rawPartition.replicaNodes
                         };
                     }
-                    topics.set(name, { id, partitions, partitionsCount: rawPartitions.length });
+                    this.#metadata.topics.set(name, { id, partitions, partitionsCount: rawPartitions.length, lastUpdate });
                 }
-                this.#metadata = {
-                    id: metadata.clusterId,
-                    brokers,
-                    topics,
-                    lastUpdate: Date.now()
+                // Now build the object to return
+                const updatedMetadata = {
+                    ...this.#metadata,
+                    topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
                 };
-                this.emitWithDebug('client', 'metadata', this.#metadata);
-                deduplicateCallback(null, this.#metadata);
+                this.emitWithDebug('client', 'metadata', updatedMetadata);
+                deduplicateCallback(null, updatedMetadata);
             }, 0);
         }, callback);
     }

package/dist/clients/base/index.d.ts

@@ -1,3 +1,3 @@
-export { Base } from './base.ts';
+export { Base, kCheckNotClosed, kClearMetadata, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kParseBroker, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from './base.ts';
 export * from './options.ts';
 export * from './types.ts';

package/dist/clients/base/index.js

@@ -1,3 +1,3 @@
-export { Base } from "./base.js";
+export { Base, kCheckNotClosed, kClearMetadata, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kParseBroker, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "./base.js";
 export * from "./options.js";
 export * from "./types.js";

package/dist/clients/base/options.d.ts

@@ -78,6 +78,11 @@ export declare const baseOptionsSchema: {
             type: string;
             additionalProperties: boolean;
         };
+        tlsServerName: {
+            oneOf: {
+                type: string;
+            }[];
+        };
         sasl: {
             type: string;
             properties: {

package/dist/clients/base/options.js

@@ -33,6 +33,7 @@ export const baseOptionsSchema = {
         retryDelay: { type: 'number', minimum: 0 },
         maxInflights: { type: 'number', minimum: 0 },
         tls: { type: 'object', additionalProperties: true }, // No validation as they come from Node.js
+        tlsServerName: { oneOf: [{ type: 'boolean' }, { type: 'string' }] },
         sasl: {
             type: 'object',
             properties: {

package/dist/clients/base/types.d.ts

@@ -14,6 +14,7 @@ export interface ClusterTopicMetadata {
     id: string;
     partitions: ClusterPartitionMetadata[];
     partitionsCount: number;
+    lastUpdate: number;
 }
 export interface ClusterMetadata {
     id: string;

package/dist/clients/consumer/consumer.d.ts

@@ -4,7 +4,7 @@ import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { Base, kFetchConnections } from '../base/base.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
-import { type CommitOptions, type ConsumeOptions, type ConsumerOptions, type FetchOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets } from './types.ts';
+import { type CommitOptions, type ConsumeOptions, type ConsumerOptions, type FetchOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
 export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ConsumerOptions<Key, Value, HeaderKey, HeaderValue>> {
     #private;
     groupId: string;
@@ -25,6 +25,8 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     commit(options: CommitOptions): Promise<void>;
     listOffsets(options: ListOffsetsOptions, callback: CallbackWithPromise<Offsets>): void;
     listOffsets(options: ListOffsetsOptions): Promise<Offsets>;
+    listOffsetsWithTimestamps(options: ListOffsetsOptions, callback: CallbackWithPromise<OffsetsWithTimestamps>): void;
+    listOffsetsWithTimestamps(options: ListOffsetsOptions): Promise<OffsetsWithTimestamps>;
     listCommittedOffsets(options: ListCommitsOptions, callback: CallbackWithPromise<Offsets>): void;
     listCommittedOffsets(options: ListCommitsOptions): Promise<Offsets>;
     findGroupCoordinator(callback: CallbackWithPromise<number>): void;

package/dist/clients/consumer/consumer.js

@@ -9,6 +9,7 @@ import { defaultBaseOptions } from "../base/options.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
 import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
+import { roundRobinAssigner } from "./partitions-assigners.js";
 import { TopicsMap } from "./topics-map.js";
 export class Consumer extends Base {
     groupId;
@@ -23,6 +24,7 @@ export class Consumer extends Base {
     #coordinatorId;
     #heartbeatInterval;
     #streams;
+    #partitionsAssigner;
     /*
       The following requests are blocking in Kafka:
   
@@ -56,6 +58,7 @@ export class Consumer extends Base {
         this.#coordinatorId = null;
         this.#heartbeatInterval = null;
         this.#streams = new Set();
+        this.#partitionsAssigner = this[kOptions].partitionAssigner ?? roundRobinAssigner;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
         // Initialize connection pool
         this[kFetchConnections] = this[kCreateConnectionPool]();
@@ -176,7 +179,22 @@ export class Consumer extends Base {
             callback(validationError, undefined);
             return callback[kCallbackPromise];
         }
-        consumerOffsetsChannel.traceCallback(this.#listOffsets, 1, createDiagnosticContext({ client: this, operation: 'listOffsets', options }), this, options, callback);
+        consumerOffsetsChannel.traceCallback(this.#listOffsets, 2, createDiagnosticContext({ client: this, operation: 'listOffsets', options }), this, false, options, callback);
+        return callback[kCallbackPromise];
+    }
+    listOffsetsWithTimestamps(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        const validationError = this[kValidateOptions](options, listOffsetsOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError, undefined);
+            return callback[kCallbackPromise];
+        }
+        consumerOffsetsChannel.traceCallback(this.#listOffsets, 2, createDiagnosticContext({ client: this, operation: 'listOffsets', options }), this, true, options, callback);
         return callback[kCallbackPromise];
     }
     listCommittedOffsets(options, callback) {
@@ -309,7 +327,7 @@ export class Consumer extends Base {
             callback(error);
         });
     }
-    #listOffsets(options, callback) {
+    #listOffsets(withTimestamps, options, callback) {
         this[kMetadata]({ topics: options.topics }, (error, metadata) => {
             if (error) {
                 callback(error, undefined);
@@ -318,7 +336,12 @@ export class Consumer extends Base {
             const requests = new Map();
             for (const name of options.topics) {
                 const topic = metadata.topics.get(name);
+                const toInclude = options.partitions?.[name] ?? [];
+                const hasPartitionsFilter = toInclude.length > 0;
                 for (let i = 0; i < topic.partitionsCount; i++) {
+                    if (hasPartitionsFilter && !toInclude.includes(i)) {
+                        continue;
+                    }
                     const partition = topic.partitions[i];
                     const { leader, leaderEpoch } = partition;
                     let leaderRequests = requests.get(leader);
@@ -359,16 +382,34 @@ export class Consumer extends Base {
                     callback(error, undefined);
                     return;
                 }
-                const offsets = new Map();
-                for (const response of responses) {
-                    for (const { name: topic, partitions } of response.topics) {
-                        let topicOffsets = offsets.get(topic);
-                        if (!topicOffsets) {
-                            topicOffsets = Array(metadata.topics.get(topic).partitionsCount);
-                            offsets.set(topic, topicOffsets);
+                let offsets = new Map();
+                if (withTimestamps) {
+                    offsets = new Map();
+                    for (const response of responses) {
+                        for (const { name: topic, partitions } of response.topics) {
+                            let topicOffsets = offsets.get(topic);
+                            if (!topicOffsets) {
+                                topicOffsets = new Map();
+                                offsets.set(topic, topicOffsets);
+                            }
+                            for (const { partitionIndex: index, offset, timestamp } of partitions) {
+                                topicOffsets.set(index, { offset, timestamp });
+                            }
                         }
-                        for (const { partitionIndex: index, offset } of partitions) {
-                            topicOffsets[index] = offset;
+                    }
+                }
+                else {
+                    offsets = new Map();
+                    for (const response of responses) {
+                        for (const { name: topic, partitions } of response.topics) {
+                            let topicOffsets = offsets.get(topic);
+                            if (!topicOffsets) {
+                                topicOffsets = Array(metadata.topics.get(topic).partitionsCount);
+                                offsets.set(topic, topicOffsets);
+                            }
+                            for (const { partitionIndex: index, offset } of partitions) {
+                                topicOffsets[index] = offset;
+                            }
                         }
                     }
                 }
@@ -684,7 +725,7 @@ export class Consumer extends Base {
                         callback(error, undefined);
                         return;
                     }
-                    this.#performSyncGroup(this.#roundRobinAssignments(metadata), callback);
+                    this.#performSyncGroup(this.#createAssignments(metadata), callback);
                 });
                 return;
             }
@@ -785,7 +826,7 @@ export class Consumer extends Base {
             w.appendString(topic).appendArray(partitions, (w, a) => w.appendInt32(a), true, false);
         }, true, false).buffer;
     }
-    #roundRobinAssignments(metadata) {
+    #createAssignments(metadata) {
         const partitionTracker = new Map();
         // First of all, layout topics-partitions in a list
         for (const [topic, partitions] of metadata.topics) {
@@ -805,37 +846,14 @@ export class Consumer extends Base {
             }
             return [{ memberId: this.memberId, assignment: this.#encodeProtocolAssignment(assignments) }];
         }
-        // Flat the list of members and subscribed topics
-        const members = [];
-        const subscribedTopics = new Set();
-        for (const [memberId, subscription] of this.#members) {
-            members.push({ memberId, assignments: new Map() });
-            for (const topic of subscription.topics) {
-                subscribedTopics.add(topic);
-            }
-        }
-        // Assign topic-partitions in round robin
-        let currentMember = 0;
-        for (const topic of subscribedTopics) {
-            const partitionsCount = metadata.topics.get(topic).partitionsCount;
-            for (let i = 0; i < partitionsCount; i++) {
-                const member = members[currentMember++ % membersSize];
-                let topicAssignments = member.assignments.get(topic);
-                if (!topicAssignments) {
-                    topicAssignments = { topic, partitions: [] };
-                    member.assignments.set(topic, topicAssignments);
-                }
-                topicAssignments?.partitions.push(i);
-            }
-        }
-        const assignments = [];
-        for (const member of members) {
-            assignments.push({
+        const encodedAssignments = [];
+        for (const member of this.#partitionsAssigner(this.memberId, this.#members, new Set(this.topics.current), metadata)) {
+            encodedAssignments.push({
                 memberId: member.memberId,
                 assignment: this.#encodeProtocolAssignment(Array.from(member.assignments.values()))
             });
         }
-        return assignments;
+        return encodedAssignments;
     }
     #getRejoinError(error) {
         const protocolError = error.findBy?.('needsRejoin', true);

package/dist/clients/consumer/options.d.ts

@@ -43,6 +43,9 @@ export declare const groupOptionsProperties: {
             };
         };
     };
+    partitionAssigner: {
+        function: boolean;
+    };
 };
 export declare const groupOptionsAdditionalValidations: {
     rebalanceTimeout: {
@@ -161,6 +164,9 @@ export declare const groupOptionsSchema: {
                 };
             };
         };
+        partitionAssigner: {
+            function: boolean;
+        };
     };
     additionalProperties: boolean;
 };
@@ -257,6 +263,9 @@ export declare const consumeOptionsSchema: {
                 };
             };
         };
+        partitionAssigner: {
+            function: boolean;
+        };
         topics: {
             type: string;
             items: {
@@ -393,6 +402,9 @@ export declare const consumerOptionsSchema: {
                 };
             };
         };
+        partitionAssigner: {
+            function: boolean;
+        };
         groupId: {
             type: string;
             pattern: string;
@@ -494,6 +506,9 @@ export declare const fetchOptionsSchema: {
                 };
             };
         };
+        partitionAssigner: {
+            function: boolean;
+        };
         node: {
             type: string;
             minimum: number;
@@ -607,6 +622,16 @@ export declare const listOffsetsOptionsSchema: {
                 pattern: string;
             };
         };
+        partitions: {
+            type: string;
+            additionalProperties: {
+                type: string;
+                items: {
+                    type: string;
+                    minimum: number;
+                };
+            };
+        };
         isolationLevel: {
             type: string;
             enum: string[];

package/dist/clients/consumer/options.js

@@ -21,7 +21,8 @@ export const groupOptionsProperties = {
                 metadata: { oneOf: [{ type: 'string' }, { buffer: true }] }
             }
         }
-    }
+    },
+    partitionAssigner: { function: true }
 };
 export const groupOptionsAdditionalValidations = {
     rebalanceTimeout: {
@@ -178,6 +179,13 @@ export const listOffsetsOptionsSchema = {
     type: 'object',
     properties: {
         topics: { type: 'array', items: idProperty },
+        partitions: {
+            type: 'object',
+            additionalProperties: {
+                type: 'array',
+                items: { type: 'number', minimum: 0 }
+            }
+        },
         isolationLevel: { type: 'string', enum: Object.keys(FetchIsolationLevels) },
         timestamp: { bigint: true }
     },

package/dist/clients/consumer/partitions-assigners.d.ts

@@ -0,0 +1,3 @@
+import { type ClusterMetadata } from '../base/types.ts';
+import { type ExtendedGroupProtocolSubscription, type GroupPartitionsAssignments } from './types.ts';
+export declare function roundRobinAssigner(_current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata): GroupPartitionsAssignments[];

package/dist/clients/consumer/partitions-assigners.js

@@ -0,0 +1,23 @@
+export function roundRobinAssigner(_current, members, topics, metadata) {
+    const membersSize = members.size;
+    const assignments = [];
+    // Flat the list of members and subscribed topics
+    for (const memberId of members.keys()) {
+        assignments.push({ memberId, assignments: new Map() });
+    }
+    // Assign topic-partitions in round robin
+    let currentMember = 0;
+    for (const topic of topics) {
+        const partitionsCount = metadata.topics.get(topic).partitionsCount;
+        for (let i = 0; i < partitionsCount; i++) {
+            const member = assignments[currentMember++ % membersSize];
+            let topicAssignments = member.assignments.get(topic);
+            if (!topicAssignments) {
+                topicAssignments = { topic, partitions: [] };
+                member.assignments.set(topic, topicAssignments);
+            }
+            topicAssignments?.partitions.push(i);
+        }
+    }
+    return assignments;
+}

package/dist/clients/consumer/types.d.ts

@@ -1,7 +1,7 @@
 import { type FetchRequestTopic } from '../../apis/consumer/fetch-v17.ts';
 import { type FetchIsolationLevel } from '../../apis/enumerations.ts';
 import { type KafkaRecord, type Message } from '../../protocol/records.ts';
-import { type BaseOptions, type TopicWithPartitionAndOffset } from '../base/types.ts';
+import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from '../base/types.ts';
 import { type Deserializers } from '../serde.ts';
 export interface GroupProtocolSubscription {
     name: string;
@@ -12,12 +12,21 @@ export interface GroupAssignment {
     topic: string;
     partitions: number[];
 }
+export interface GroupPartitionsAssignments {
+    memberId: string;
+    assignments: Map<string, GroupAssignment>;
+}
 export interface ExtendedGroupProtocolSubscription extends Omit<GroupProtocolSubscription, 'name'> {
     topics?: string[];
     memberId: string;
 }
 export type Offsets = Map<string, bigint[]>;
+export type OffsetsWithTimestamps = Map<string, Map<number, {
+    offset: bigint;
+    timestamp: bigint;
+}>>;
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
+export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
 export declare const MessagesStreamModes: {
     readonly LATEST: "latest";
     readonly EARLIEST: "earliest";
@@ -38,6 +47,7 @@ export interface GroupOptions {
     rebalanceTimeout?: number;
     heartbeatInterval?: number;
     protocols?: GroupProtocolSubscription[];
+    partitionAssigner?: GroupPartitionsAssigner;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
     autocommit?: boolean | number;
@@ -74,6 +84,7 @@ export interface ListCommitsOptions {
 }
 export interface ListOffsetsOptions {
     topics: string[];
+    partitions?: Record<string, number[]>;
     timestamp?: bigint;
     isolationLevel?: FetchIsolationLevel;
 }

package/dist/clients/producer/options.d.ts

@@ -10,13 +10,17 @@ export declare const produceOptionsProperties: {
     };
     acks: {
         type: string;
-        enum: (0 | 1 | -1)[];
-        errorMessage: string;
+        enumeration: {
+            allowed: (0 | 1 | -1)[];
+            errorMessage: string;
+        };
     };
     compression: {
         type: string;
-        enum: string[];
-        errorMessage: string;
+        enumeration: {
+            allowed: string[];
+            errorMessage: string;
+        };
     };
     partitioner: {
         function: boolean;
@@ -42,13 +46,17 @@ export declare const produceOptionsSchema: {
         };
         acks: {
             type: string;
-            enum: (0 | 1 | -1)[];
-            errorMessage: string;
+            enumeration: {
+                allowed: (0 | 1 | -1)[];
+                errorMessage: string;
+            };
         };
         compression: {
             type: string;
-            enum: string[];
-            errorMessage: string;
+            enumeration: {
+                allowed: string[];
+                errorMessage: string;
+            };
         };
         partitioner: {
             function: boolean;
@@ -78,13 +86,17 @@ export declare const sendOptionsSchema: {
         };
         acks: {
             type: string;
-            enum: (0 | 1 | -1)[];
-            errorMessage: string;
+            enumeration: {
+                allowed: (0 | 1 | -1)[];
+                errorMessage: string;
+            };
         };
         compression: {
             type: string;
-            enum: string[];
-            errorMessage: string;
+            enumeration: {
+                allowed: string[];
+                errorMessage: string;
+            };
         };
         partitioner: {
             function: boolean;

package/dist/clients/producer/options.js

@@ -9,13 +9,17 @@ export const produceOptionsProperties = {
     idempotent: { type: 'boolean' },
     acks: {
         type: 'number',
-        enum: Object.values(ProduceAcks),
-        errorMessage: enumErrorMessage(ProduceAcks)
+        enumeration: {
+            allowed: Object.values(ProduceAcks),
+            errorMessage: enumErrorMessage(ProduceAcks)
+        }
     },
     compression: {
         type: 'string',
-        enum: Object.keys(compressionsAlgorithms),
-        errorMessage: enumErrorMessage(compressionsAlgorithms, true)
+        enumeration: {
+            allowed: Object.keys(compressionsAlgorithms),
+            errorMessage: enumErrorMessage(compressionsAlgorithms, true)
+        }
     },
     partitioner: { function: true },
     autocreateTopics: { type: 'boolean' },

package/dist/clients/producer/producer.d.ts

@@ -1,6 +1,7 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { Base } from '../base/base.ts';
 import { type ProduceOptions, type ProduceResult, type ProducerInfo, type ProducerOptions, type SendOptions } from './types.ts';
+export declare function noopSerializer(data?: Buffer): Buffer | undefined;
 export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ProducerOptions<Key, Value, HeaderKey, HeaderValue>> {
     #private;
     constructor(options: ProducerOptions<Key, Value, HeaderKey, HeaderValue>);

package/dist/clients/producer/producer.js

@@ -9,7 +9,7 @@ import { ensureMetric } from "../metrics.js";
 import { produceOptionsValidator, producerOptionsValidator, sendOptionsValidator } from "./options.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
-function noopSerializer(data) {
+export function noopSerializer(data) {
     return data;
 }
 export class Producer extends Base {

package/dist/network/connection.d.ts

@@ -18,6 +18,7 @@ export interface ConnectionOptions {
     connectTimeout?: number;
     maxInflights?: number;
     tls?: TLSConnectionOptions;
+    tlsServerName?: string | boolean;
     sasl?: SASLOptions;
     ownerId?: number;
 }

package/dist/network/connection.js

@@ -99,6 +99,10 @@ export class Connection extends EventEmitter {
             const connectionOptions = {
                 timeout: this.#options.connectTimeout
             };
+            if (this.#options.tlsServerName) {
+                connectionOptions.servername =
+                    typeof this.#options.tlsServerName === 'string' ? this.#options.tlsServerName : host;
+            }
             const connectionTimeoutHandler = () => {
                 const error = new TimeoutError(`Connection to ${host}:${port} timed out.`);
                 diagnosticContext.error = error;
@@ -114,14 +118,14 @@ export class Connection extends EventEmitter {
                 this.#onConnectionError(host, port, diagnosticContext, error);
             };
             this.emit('connecting');
-            /* c8 ignore next 3 - TLS connection is not tested but we rely on Node.js tests */
             this.#host = host;
             this.#port = port;
+            /* c8 ignore next 3 - TLS connection is not tested but we rely on Node.js tests */
             this.#socket = this.#options.tls
                 ? createTLSConnection(port, host, { ...this.#options.tls, ...connectionOptions })
                 : createConnection({ ...connectionOptions, port, host });
             this.#socket.setNoDelay(true);
-            this.#socket.once('connect', () => {
+            this.#socket.once(this.#options.tls ? 'secureConnect' : 'connect', () => {
                 this.#socket.removeListener('timeout', connectionTimeoutHandler);
                 this.#socket.removeListener('error', connectionErrorHandler);
                 this.#socket.on('error', this.#onError.bind(this));

package/dist/protocol/compression.js

@@ -3,7 +3,7 @@ import zlib from 'node:zlib';
 import { UnsupportedCompressionError } from "../errors.js";
 import { DynamicBuffer } from "./dynamic-buffer.js";
 const require = createRequire(import.meta.url);
-// @ts-ignore
+// @ts-ignore - Added in Node.js 22.15.0
 const { zstdCompressSync, zstdDecompressSync, gzipSync, gunzipSync } = zlib;
 function ensureBuffer(data) {
     return DynamicBuffer.isDynamicBuffer(data) ? data.slice() : data;

package/dist/protocol/reader.js

@@ -208,7 +208,10 @@ export class Reader {
         return this.readNullableBytes(compact) || EMPTY_BUFFER;
     }
     readVarIntBytes() {
-        const length = this.readVarInt();
+        let length = this.readVarInt();
+        if (length === -1) {
+            length = 0;
+        }
         const value = this.buffer.slice(this.position, this.position + length);
         this.position += length;
         return value;

package/dist/utils.d.ts

@@ -1,6 +1,13 @@
 import { Ajv2020 } from 'ajv/dist/2020.js';
 import debug from 'debug';
 import { type DynamicBuffer } from './protocol/dynamic-buffer.ts';
+export interface EnumerationDefinition<T> {
+    allowed: T[];
+    errorMessage?: string;
+}
+export type KeywordSchema<T> = {
+    schema: T;
+};
 export interface DataValidationContext {
     parentData: {
         [k: string | number]: any;

package/dist/utils.js

@@ -1,4 +1,3 @@
-import ajvErrors from 'ajv-errors';
 import { Ajv2020 } from 'ajv/dist/2020.js';
 import debug from 'debug';
 import { inspect } from 'node:util';
@@ -12,8 +11,6 @@ export const loggers = {
     'consumer:heartbeat': debug('plt:kafka:consumer:heartbeat'),
     admin: debug('plt:kafka:admin')
 };
-// @ts-ignore
-ajvErrors(ajv);
 let debugDumpLogger = console.error;
 ajv.addKeyword({
     keyword: 'bigint',
@@ -75,6 +72,17 @@ ajv.addKeyword({
         }
     }
 });
+ajv.addKeyword({
+    keyword: 'enumeration', // This mimics the enum keyword but defines a custom error message
+    validate(property, current) {
+        return property.allowed.includes(current);
+    },
+    error: {
+        message({ schema }) {
+            return schema.errorMessage;
+        }
+    }
+});
 export class NumericMap extends Map {
     getWithDefault(key, fallback) {
         return this.get(key) ?? fallback;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.6.0",
+  "version": "1.8.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -26,7 +26,6 @@
   "types": "./dist/index.d.ts",
   "dependencies": {
     "ajv": "^8.17.1",
-    "ajv-errors": "^3.0.0",
     "debug": "^4.4.0",
     "fastq": "^1.19.1",
     "mnemonist": "^0.40.3",
@@ -42,6 +41,7 @@
     "@types/node": "^22.13.5",
     "@types/semver": "^7.7.0",
     "@watchable/unpromise": "^1.0.2",
+    "avsc": "^5.7.7",
     "c8": "^10.1.3",
     "cleaner-spec-reporter": "^0.5.0",
     "cronometro": "^5.3.0",
@@ -53,6 +53,7 @@
     "node-rdkafka": "^3.3.1",
     "parse5": "^7.2.1",
     "prettier": "^3.5.3",
+    "prettier-plugin-space-before-function-paren": "^0.0.8",
     "prom-client": "^15.1.3",
     "semver": "^7.7.1",
     "table": "^6.9.0",