@platformatic/kafka 1.25.0 → 1.26.0

package/dist/clients/base/base.d.ts

@@ -17,7 +17,6 @@ export declare const kGetConnection: unique symbol;
 export declare const kGetBootstrapConnection: unique symbol;
 export declare const kOptions: unique symbol;
 export declare const kConnections: unique symbol;
-export declare const kFetchConnections: unique symbol;
 export declare const kCreateConnectionPool: unique symbol;
 export declare const kClosed: unique symbol;
 export declare const kListApis: unique symbol;

package/dist/clients/base/base.js

@@ -17,7 +17,6 @@ export const kGetConnection = Symbol('plt.kafka.base.getConnection');
 export const kGetBootstrapConnection = Symbol('plt.kafka.base.getBootstrapConnection');
 export const kOptions = Symbol('plt.kafka.base.options');
 export const kConnections = Symbol('plt.kafka.base.connections');
-export const kFetchConnections = Symbol('plt.kafka.base.fetchCnnections');
 export const kCreateConnectionPool = Symbol('plt.kafka.base.createConnectionPool');
 export const kClosed = Symbol('plt.kafka.base.closed');
 export const kListApis = Symbol('plt.kafka.base.listApis');

package/dist/clients/base/options.d.ts

@@ -82,6 +82,10 @@ export declare const baseOptionsSchema: {
             type: string;
             additionalProperties: boolean;
         };
+        ssl: {
+            type: string;
+            additionalProperties: boolean;
+        };
         tlsServerName: {
             oneOf: {
                 type: string;

package/dist/clients/base/options.js

@@ -33,6 +33,7 @@ export const baseOptionsSchema = {
         maxInflights: { type: 'number', minimum: 0 },
         handleBackPressure: { type: 'boolean', default: false },
         tls: { type: 'object', additionalProperties: true }, // No validation as they come from Node.js
+        ssl: { type: 'object', additionalProperties: true }, // Alias for tls, no validation as they come from Node.js
         tlsServerName: { oneOf: [{ type: 'boolean' }, { type: 'string' }] },
         sasl: {
             type: 'object',

package/dist/clients/consumer/consumer.d.ts

@@ -1,7 +1,6 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type FetchResponse } from '../../apis/consumer/fetch-v17.ts';
-import { type ConnectionPool } from '../../network/connection-pool.ts';
-import { Base, type BaseEvents, kFetchConnections } from '../base/base.ts';
+import { Base, type BaseEvents } from '../base/base.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
 import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
@@ -20,11 +19,11 @@ export interface ConsumerEvents extends BaseEvents {
 export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ConsumerOptions<Key, Value, HeaderKey, HeaderValue>, ConsumerEvents> {
     #private;
     groupId: string;
+    groupInstanceId: string | null;
     generationId: number;
     memberId: string | null;
     topics: TopicsMap;
     assignments: GroupAssignment[] | null;
-    [kFetchConnections]: ConnectionPool;
     constructor(options: ConsumerOptions<Key, Value, HeaderKey, HeaderValue>);
     get streamsCount(): number;
     get lastHeartbeat(): Date | null;

package/dist/clients/consumer/consumer.js

@@ -7,7 +7,7 @@ import { Reader } from "../../protocol/reader.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
 import { kAutocommit, kRefreshOffsetsAndFetch } from "../../symbols.js";
-import { Base, kAfterCreate, kCheckNotClosed, kClosed, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
+import { Base, kAfterCreate, kCheckNotClosed, kClosed, kConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
 import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
@@ -15,6 +15,7 @@ import { roundRobinAssigner } from "./partitions-assigners.js";
 import { TopicsMap } from "./topics-map.js";
 export class Consumer extends Base {
     groupId;
+    groupInstanceId;
     generationId;
     memberId;
     topics;
@@ -33,21 +34,6 @@ export class Consumer extends Base {
     #groupRemoteAssignor;
     #streams;
     #lagMonitoring;
-    /*
-      The following requests are blocking in Kafka:
-  
-      FetchRequest (soprattutto con maxWaitMs)
-      JoinGroupRequest
-      SyncGroupRequest
-      OffsetCommitRequest
-      ProduceRequest
-      ListOffsetsRequest
-      ListGroupsRequest
-      DescribeGroupsRequest
-  
-      In order to avoid consumer group problems, we separate FetchRequest only on a separate connection.
-    */
-    [kFetchConnections];
     // Metrics
     #metricActiveStreams;
     #metricLags;
@@ -55,6 +41,7 @@ export class Consumer extends Base {
         super({ ...defaultConsumerOptions, ...options });
         this[kValidateOptions](options, consumerOptionsValidator, '/options');
         this.groupId = options.groupId;
+        this.groupInstanceId = options.groupInstanceId ?? null;
         this.generationId = 0;
         this.memberId = null;
         this.topics = new TopicsMap();
@@ -74,8 +61,6 @@ export class Consumer extends Base {
         this.#useConsumerGroupProtocol = this[kOptions].groupProtocol === 'consumer';
         this.#groupRemoteAssignor = this[kOptions].groupRemoteAssignor ?? null;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
-        // Initialize connection pool
-        this[kFetchConnections] = this[kCreateConnectionPool]();
         if (this[kPrometheus]) {
             ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').inc();
             this.#metricActiveStreams = ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers_streams', 'Number of active Kafka consumers streams');
@@ -125,24 +110,17 @@ export class Consumer extends Base {
                 callback(error);
                 return;
             }
-            this[kFetchConnections].close(error => {
+            super.close(error => {
                 if (error) {
                     this[kClosed] = false;
                     callback(error);
                     return;
                 }
-                super.close(error => {
-                    if (error) {
-                        this[kClosed] = false;
-                        callback(error);
-                        return;
-                    }
-                    this.topics.clear();
-                    if (this[kPrometheus]) {
-                        ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').dec();
-                    }
-                    callback(null);
-                });
+                this.topics.clear();
+                if (this[kPrometheus]) {
+                    ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').dec();
+                }
+                callback(null);
             });
         });
         return callback[kCallbackPromise];
@@ -410,7 +388,8 @@ export class Consumer extends Base {
                     retryCallback(new UserError(`Cannot find broker with node id ${options.node}`));
                     return;
                 }
-                this[kFetchConnections].get(broker, (error, connection) => {
+                const pool = options.connectionPool ?? this[kConnections];
+                pool.get(broker, (error, connection) => {
                     if (error) {
                         // When a connection was not available (either interrupted or not available) we
                         // reset the leader epoch in the options so that when connection is re-established again we can continue
@@ -638,7 +617,7 @@ export class Consumer extends Base {
                     groupCallback(error);
                     return;
                 }
-                api(connection, this.groupId, this.generationId, this.memberId, null, groupCallback);
+                api(connection, this.groupId, this.generationId, this.memberId, this.groupInstanceId, groupCallback);
             });
         }, error => {
             // The heartbeat has been aborted elsewhere, ignore the response
@@ -685,8 +664,7 @@ export class Consumer extends Base {
                     groupCallback(error);
                     return;
                 }
-                api(connection, this.groupId, this.memberId || '', this.#memberEpoch, null, // instanceId
-                null, // rackId
+                api(connection, this.groupId, this.memberId || '', this.#memberEpoch, this.groupInstanceId, null, // rackId
                 options.rebalanceTimeout, this.topics.current, this.#groupRemoteAssignor, this.#assignments, groupCallback);
             });
         }, (error, response) => {
@@ -836,8 +814,7 @@ export class Consumer extends Base {
                     return;
                 }
                 api(connection, this.groupId, this.memberId, -1, // memberEpoch = -1 signals leave
-                null, // instanceId
-                null, // rackId
+                this.groupInstanceId, null, // rackId
                 0, // rebalanceTimeout
                 [], // subscribedTopicNames
                 this.#groupRemoteAssignor, [], // topicPartitions
@@ -941,7 +918,7 @@ export class Consumer extends Base {
                     groupCallback(error);
                     return;
                 }
-                api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', null, 'consumer', protocols, '', groupCallback);
+                api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', this.groupInstanceId, 'consumer', protocols, '', groupCallback);
             });
         }, (error, response) => {
             if (!this.#membershipActive) {
@@ -1091,7 +1068,7 @@ export class Consumer extends Base {
                     groupCallback(error);
                     return;
                 }
-                api(connection, this.groupId, this.generationId, this.memberId, null, 'consumer', this.#protocol, assignments, groupCallback);
+                api(connection, this.groupId, this.generationId, this.memberId, this.groupInstanceId, 'consumer', this.#protocol, assignments, groupCallback);
             });
         }, (error, response) => {
             if (!this.#membershipActive) {

package/dist/clients/consumer/messages-stream.d.ts

@@ -1,8 +1,9 @@
 import { Readable } from 'node:stream';
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
+import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { type Message } from '../../protocol/records.ts';
 import { kAutocommit, kInstance, kRefreshOffsetsAndFetch } from '../../symbols.ts';
-import { kInspect } from '../base/base.ts';
+import { kConnections, kInspect } from '../base/base.ts';
 import { type Consumer } from './consumer.ts';
 import { type CommitOptionsPartition, type ConsumeOptions } from './types.ts';
 export declare function noopDeserializer(data?: Buffer): Buffer | undefined;
@@ -10,6 +11,7 @@ export declare function defaultCorruptedMessageHandler(): boolean;
 export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends Readable {
     #private;
     [kInstance]: number;
+    [kConnections]: ConnectionPool;
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
     get consumer(): Consumer<Key, Value, HeaderKey, HeaderValue>;
     get offsetsToFetch(): Map<string, bigint>;

package/dist/clients/consumer/messages-stream.js

@@ -5,7 +5,7 @@ import { consumerReceivesChannel, createDiagnosticContext, notifyCreation } from
 import { UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { kAutocommit, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
-import { kInspect, kPrometheus } from "../base/base.js";
+import { kConnections, kCreateConnectionPool, kInspect, kPrometheus } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { defaultConsumerOptions } from "./options.js";
 import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
@@ -18,6 +18,18 @@ export function noopDeserializer(data) {
 export function defaultCorruptedMessageHandler() {
     return true;
 }
+function messageToJSON() {
+    return {
+        key: this.key,
+        value: this.value,
+        headers: Array.from(this.headers.entries()),
+        topic: this.topic,
+        partition: this.partition,
+        timestamp: this.timestamp.toString(),
+        offset: this.offset.toString(),
+        metadata: this.metadata
+    };
+}
 let currentInstance = 0;
 export class MessagesStream extends Readable {
     #consumer;
@@ -40,11 +52,27 @@ export class MessagesStream extends Readable {
     #autocommitEnabled;
     #autocommitInterval;
     #autocommitInflight;
-    #shouldClose;
+    #closed;
     #closeCallbacks;
     #metricsConsumedMessages;
     #corruptedMessageHandler;
     [kInstance];
+    /*
+      The following requests are blocking in Kafka:
+  
+      FetchRequest (soprattutto con maxWaitMs)
+      JoinGroupRequest
+      SyncGroupRequest
+      OffsetCommitRequest
+      ProduceRequest
+      ListOffsetsRequest
+      ListGroupsRequest
+      DescribeGroupsRequest
+  
+      In order to avoid consumer group problems, we separate FetchRequest only on a separate connection.
+      Also, to avoid head-of-line blocking between streams, we use a pool of connections for each stream.
+    */
+    [kConnections];
     constructor(consumer, options) {
         const { autocommit, mode, fallbackMode, maxFetches, offsets, deserializers, onCorruptedMessage, 
         // The options below are only destructured to avoid being part of structuredClone below
@@ -61,6 +89,7 @@ export class MessagesStream extends Readable {
             highWaterMark: maxFetches ?? options.highWaterMark ?? defaultConsumerOptions.highWaterMark
         });
         this[kInstance] = currentInstance++;
+        this[kConnections] = consumer[kCreateConnectionPool]();
         this.#consumer = consumer;
         this.#mode = mode ?? MessagesStreamModes.LATEST;
         this.#fallbackMode = fallbackMode ?? MessagesStreamFallbackModes.LATEST;
@@ -78,7 +107,7 @@ export class MessagesStream extends Readable {
         this.#headerValueDeserializer = deserializers?.headerValue ?? noopDeserializer;
         this.#autocommitEnabled = !!options.autocommit;
         this.#autocommitInflight = false;
-        this.#shouldClose = false;
+        this.#closed = false;
         this.#closeCallbacks = [];
         this.#corruptedMessageHandler = onCorruptedMessage ?? defaultCorruptedMessageHandler;
         // Restore offsets
@@ -150,11 +179,11 @@ export class MessagesStream extends Readable {
             return callback[kCallbackPromise];
         }
         this.#closeCallbacks.push(callback);
-        if (this.#shouldClose) {
-            this.#invokeCloseCallbacks(null);
+        if (this.#closed) {
+            this.#afterClose(null);
             return callback[kCallbackPromise];
         }
-        this.#shouldClose = true;
+        this.#closed = true;
         this.push(null);
         if (this.#autocommitInterval) {
             clearInterval(this.#autocommitInterval);
@@ -166,32 +195,33 @@ export class MessagesStream extends Readable {
         }
         /* c8 ignore next 3 - Hard to test */
         this.once('error', error => {
-            callback(error);
+            this.#afterClose(error);
         });
         this.once('close', () => {
             // We have offsets that were enqueued to be committed. Perform the operation
+            /* c8 ignore next 3 - Hard to test */
             if (this.#offsetsToCommit.size > 0) {
                 this[kAutocommit]();
             }
             // We have offsets that are being committed. These are awaited despite of the force parameters
             if (this.#autocommitInflight) {
                 this.once('autocommit', error => {
-                    this.#invokeCloseCallbacks(error);
+                    this.#afterClose(error);
                 });
                 return;
             }
-            this.#invokeCloseCallbacks(null);
+            this.#afterClose(null);
         });
         return callback[kCallbackPromise];
     }
     isActive() {
-        if (this.#shouldClose || this.closed || this.destroyed) {
+        if (this.#closed || this.closed || this.destroyed) {
             return false;
         }
         return this.#consumer.isActive();
     }
     isConnected() {
-        if (this.#shouldClose || this.closed || this.destroyed) {
+        if (this.#closed || this.closed || this.destroyed) {
             return false;
         }
         return this.#consumer.isConnected();
@@ -243,7 +273,7 @@ export class MessagesStream extends Readable {
     }
     #fetch() {
         /* c8 ignore next 4 - Hard to test */
-        if (this.#shouldClose || this.closed || this.destroyed) {
+        if (this.#closed || this.closed || this.destroyed) {
             this.push(null);
             return;
         }
@@ -256,7 +286,7 @@ export class MessagesStream extends Readable {
                 this.emit('fetch');
                 // The stream has been closed, ignore any error
                 /* c8 ignore next 4 - Hard to test */
-                if (this.#shouldClose || this.closed || this.destroyed) {
+                if (this.#closed || this.closed || this.destroyed) {
                     this.push(null);
                     return;
                 }
@@ -264,7 +294,7 @@ export class MessagesStream extends Readable {
                 return;
             }
             /* c8 ignore next 5 - Hard to test */
-            if (this.#shouldClose || this.closed || this.destroyed) {
+            if (this.#closed || this.closed || this.destroyed) {
                 this.emit('fetch');
                 this.push(null);
                 return;
@@ -317,14 +347,14 @@ export class MessagesStream extends Readable {
                     if (error) {
                         // The stream has been closed, ignore the error
                         /* c8 ignore next 4 - Hard to test */
-                        if (this.#shouldClose || this.closed || this.destroyed) {
+                        if (this.#closed || this.closed || this.destroyed) {
                             this.push(null);
                             return;
                         }
                         this.destroy(error);
                         return;
                     }
-                    if (this.#shouldClose || this.closed || this.destroyed) {
+                    if (this.#closed || this.closed || this.destroyed) {
                         // When it's the last inflight, we finally close the stream.
                         // This is done to avoid the user exiting from consmuming metrics like for-await and still see the process up.
                         if (this.#inflightNodes.size === 0) {
@@ -421,7 +451,8 @@ export class MessagesStream extends Readable {
                                 timestamp: firstTimestamp + record.timestampDelta,
                                 offset,
                                 commit,
-                                metadata: messageMetadata
+                                metadata: messageMetadata,
+                                toJSON: messageToJSON
                             };
                             diagnosticContext.result = message;
                             consumerReceivesChannel.asyncStart.publish(diagnosticContext);
@@ -512,7 +543,7 @@ export class MessagesStream extends Readable {
         }, (error, offsets) => {
             if (error) {
                 /* c8 ignore next 4 - Hard to test */
-                if (this.#shouldClose || this.closed || this.destroyed) {
+                if (this.#closed || this.closed || this.destroyed) {
                     callback(null);
                     return;
                 }
@@ -539,7 +570,7 @@ export class MessagesStream extends Readable {
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
                 if (error) {
                     /* c8 ignore next 4 - Hard to test */
-                    if (this.#shouldClose || this.closed || this.destroyed) {
+                    if (this.#closed || this.closed || this.destroyed) {
                         callback(null);
                         return;
                     }
@@ -594,11 +625,13 @@ export class MessagesStream extends Readable {
     #assignmentsForTopic(topic) {
         return this.#consumer.assignments?.find(assignment => assignment.topic === topic);
     }
-    #invokeCloseCallbacks(error) {
-        for (const callback of this.#closeCallbacks) {
-            callback(error);
-        }
-        this.#closeCallbacks = [];
+    #afterClose(error) {
+        this[kConnections].close(closeError => {
+            for (const callback of this.#closeCallbacks) {
+                callback(error ?? closeError);
+            }
+            this.#closeCallbacks = [];
+        });
     }
     /* c8 ignore next 3 - This is a private API used to debug during development */
     [kInspect](...args) {

package/dist/clients/consumer/options.d.ts

@@ -1,4 +1,8 @@
 export declare const groupOptionsProperties: {
+    groupInstanceId: {
+        type: string;
+        pattern: string;
+    };
     sessionTimeout: {
         type: string;
         minimum: number;
@@ -127,6 +131,10 @@ export declare const consumeOptionsProperties: {
 export declare const groupOptionsSchema: {
     type: string;
     properties: {
+        groupInstanceId: {
+            type: string;
+            pattern: string;
+        };
         sessionTimeout: {
             type: string;
             minimum: number;
@@ -233,6 +241,10 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        groupInstanceId: {
+            type: string;
+            pattern: string;
+        };
         sessionTimeout: {
             type: string;
             minimum: number;
@@ -384,6 +396,10 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        groupInstanceId: {
+            type: string;
+            pattern: string;
+        };
         sessionTimeout: {
             type: string;
             minimum: number;
@@ -495,6 +511,10 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        groupInstanceId: {
+            type: string;
+            pattern: string;
+        };
         sessionTimeout: {
             type: string;
             minimum: number;
@@ -588,6 +608,9 @@ export declare const fetchOptionsSchema: {
                 required: string[];
             };
         };
+        connectionPool: {
+            type: string;
+        };
     };
     required: string[];
     additionalProperties: boolean;

package/dist/clients/consumer/options.js

@@ -4,6 +4,7 @@ import { idProperty, topicWithPartitionAndOffsetProperties } from "../base/optio
 import { serdeProperties } from "../serde.js";
 import { allowedMessagesStreamFallbackModes, allowedMessagesStreamModes } from "./types.js";
 export const groupOptionsProperties = {
+    groupInstanceId: { type: 'string', pattern: '^\\S+$' },
     sessionTimeout: { type: 'number', minimum: 0 },
     rebalanceTimeout: { type: 'number', minimum: 0 },
     heartbeatInterval: { type: 'number', minimum: 0 },
@@ -128,6 +129,7 @@ export const fetchOptionsSchema = {
                 required: ['topicId', 'partitions']
             }
         },
+        connectionPool: { type: 'object' },
         ...groupOptionsProperties,
         ...consumeOptionsProperties
     },

package/dist/clients/consumer/types.d.ts

@@ -1,5 +1,6 @@
 import { type FetchRequestTopic } from '../../apis/consumer/fetch-v17.ts';
 import { type GroupProtocols } from '../../apis/enumerations.ts';
+import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { type KafkaRecord, type Message } from '../../protocol/records.ts';
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from '../base/types.ts';
 import { type Deserializers } from '../serde.ts';
@@ -46,6 +47,7 @@ export type MessagesStreamFallbackMode = keyof typeof MessagesStreamFallbackMode
 export type MessagesStreamFallbackModeValue = (typeof MessagesStreamFallbackModes)[keyof typeof MessagesStreamFallbackModes];
 export interface GroupOptions {
     groupProtocol?: typeof GroupProtocols.CLASSIC;
+    groupInstanceId?: string;
     sessionTimeout?: number;
     rebalanceTimeout?: number;
     heartbeatInterval?: number;
@@ -54,6 +56,7 @@ export interface GroupOptions {
 }
 export interface ConsumerGroupOptions {
     groupProtocol: typeof GroupProtocols.CONSUMER;
+    groupInstanceId?: string;
     groupRemoteAssignor?: string;
     rebalanceTimeout?: number;
 }
@@ -81,6 +84,7 @@ export type ConsumerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions &
 export type FetchOptions<Key, Value, HeaderKey, HeaderValue> = Pick<ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue>, 'minBytes' | 'maxBytes' | 'maxWaitTime' | 'isolationLevel'> & {
     node: number;
     topics: FetchRequestTopic[];
+    connectionPool?: ConnectionPool;
 };
 export interface CommitOptionsPartition extends TopicWithPartitionAndOffset {
     leaderEpoch: number;

package/dist/network/connection.d.ts

@@ -4,8 +4,8 @@ import { type CallbackWithPromise } from '../apis/callbacks.ts';
 import { type Callback, type ResponseParser } from '../apis/definitions.ts';
 import { type SASLMechanismValue } from '../apis/enumerations.ts';
 import { type SaslAuthenticateResponse, type SASLAuthenticationAPI } from '../apis/security/sasl-authenticate-v2.ts';
-import { TypedEventEmitter, type TypedEvents } from '../events.ts';
 import { TimeoutError } from '../errors.ts';
+import { TypedEventEmitter, type TypedEvents } from '../events.ts';
 import { Writer } from '../protocol/writer.ts';
 export interface ConnectionEvents extends TypedEvents {
     connecting: () => void;
@@ -40,6 +40,7 @@ export interface ConnectionOptions {
     requestTimeout?: number;
     maxInflights?: number;
     tls?: TLSConnectionOptions;
+    ssl?: TLSConnectionOptions;
     tlsServerName?: string | boolean;
     sasl?: SASLOptions;
     ownerId?: number;
@@ -62,6 +63,7 @@ export declare const ConnectionStatuses: {
     readonly NONE: "none";
     readonly CONNECTING: "connecting";
     readonly AUTHENTICATING: "authenticating";
+    readonly REAUTHENTICATING: "reauthenticating";
     readonly CONNECTED: "connected";
     readonly CLOSED: "closed";
     readonly CLOSING: "closing";
@@ -89,4 +91,5 @@ export declare class Connection extends TypedEventEmitter<ConnectionEvents> {
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
     send<ReturnType>(apiKey: number, apiVersion: number, createPayload: () => Writer, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
+    reauthenticate(): void;
 }

package/dist/network/connection.js

@@ -5,8 +5,8 @@ import { createPromisifiedCallback, kCallbackPromise } from "../apis/callbacks.j
 import { allowedSASLMechanisms, SASLMechanisms } from "../apis/enumerations.js";
 import { saslAuthenticateV2, saslHandshakeV1 } from "../apis/index.js";
 import { connectionsApiChannel, connectionsConnectsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
-import { TypedEventEmitter } from "../events.js";
 import { AuthenticationError, NetworkError, TimeoutError, UnexpectedCorrelationIdError, UserError } from "../errors.js";
+import { TypedEventEmitter } from "../events.js";
 import { protocolAPIsById } from "../protocol/apis.js";
 import { EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE, INT32_SIZE } from "../protocol/definitions.js";
 import { DynamicBuffer } from "../protocol/dynamic-buffer.js";
@@ -19,6 +19,7 @@ export const ConnectionStatuses = {
     NONE: 'none',
     CONNECTING: 'connecting',
     AUTHENTICATING: 'authenticating',
+    REAUTHENTICATING: 'reauthenticating',
     CONNECTED: 'connected',
     CLOSED: 'closed',
     CLOSING: 'closing',
@@ -55,6 +56,7 @@ export class Connection extends TypedEventEmitter {
         this.setMaxListeners(0);
         this.#instanceId = currentInstance++;
         this.#options = Object.assign({}, defaultOptions, options);
+        this.#options.tls ??= this.#options.ssl;
         this.#status = ConnectionStatuses.NONE;
         this.#clientId = clientId;
         this.#ownerId = options.ownerId;
@@ -261,6 +263,17 @@ export class Connection extends TypedEventEmitter {
             return this.#sendRequest(request);
         }, this.#onResponse.bind(this, request, callback));
     }
+    reauthenticate() {
+        if (!this.#options.sasl) {
+            return;
+        }
+        const host = this.#host;
+        const port = this.#port;
+        const diagnosticContext = createDiagnosticContext({ connection: this, operation: 'reauthenticate', host, port });
+        this.#status = ConnectionStatuses.REAUTHENTICATING;
+        clearTimeout(this.#reauthenticationTimeout);
+        this.#authenticate(host, port, diagnosticContext);
+    }
     #onResponse(request, callback, error, payload) {
         clearTimeout(request.timeoutHandle);
         request.timeoutHandle = null;
@@ -314,7 +327,9 @@ export class Connection extends TypedEventEmitter {
     #sendRequest(request) {
         connectionsApiChannel.start.publish(request.diagnostic);
         try {
-            if (this.#status !== ConnectionStatuses.CONNECTED && this.#status !== ConnectionStatuses.AUTHENTICATING) {
+            if (this.#status !== ConnectionStatuses.CONNECTED &&
+                this.#status !== ConnectionStatuses.AUTHENTICATING &&
+                this.#status !== ConnectionStatuses.REAUTHENTICATING) {
                 request.callback(new NetworkError('Connection closed'), undefined);
                 return false;
             }
@@ -387,17 +402,10 @@ export class Connection extends TypedEventEmitter {
             return;
         }
         if (sessionLifetimeMs > 0) {
-            this.#reauthenticationTimeout = setTimeout(() => {
-                const diagnosticContext = createDiagnosticContext({
-                    connection: this,
-                    operation: 'reauthenticate',
-                    host,
-                    port
-                });
-                this.#authenticate(host, port, diagnosticContext);
-            }, Number(sessionLifetimeMs) * 0.8);
+            this.#reauthenticationTimeout = setTimeout(this.reauthenticate.bind(this), Number(sessionLifetimeMs) * 0.8);
         }
-        if (this.#status === ConnectionStatuses.CONNECTED) {
+        if (this.#status === ConnectionStatuses.CONNECTED || this.#status === ConnectionStatuses.REAUTHENTICATING) {
+            this.#status = ConnectionStatuses.CONNECTED;
             this.emit('sasl:authentication:extended', authBytes);
         }
         else {

package/dist/protocol/records.d.ts

@@ -24,11 +24,22 @@ export interface MessageConsumerMetadata {
     generationId: number;
     memberId: string;
 }
+export interface MessageJSON<Key = unknown, Value = unknown, HeaderKey = unknown, HeaderValue = unknown> {
+    key: Key;
+    value: Value;
+    headers: Array<[HeaderKey, HeaderValue]>;
+    topic: string;
+    partition: number;
+    timestamp: string;
+    offset: string;
+    metadata: Record<string, unknown>;
+}
 export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
     headers: Map<HeaderKey, HeaderValue>;
     offset: bigint;
     metadata: Record<string, unknown>;
     commit(callback?: (error?: Error) => void): void | Promise<void>;
+    toJSON(): MessageJSON<Key, Value, HeaderKey, HeaderValue>;
 }
 export interface MessageRecord {
     key?: Buffer;

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -17,7 +17,6 @@ export declare const kGetConnection: unique symbol;
 export declare const kGetBootstrapConnection: unique symbol;
 export declare const kOptions: unique symbol;
 export declare const kConnections: unique symbol;
-export declare const kFetchConnections: unique symbol;
 export declare const kCreateConnectionPool: unique symbol;
 export declare const kClosed: unique symbol;
 export declare const kListApis: unique symbol;

package/dist/typescript-4/dist/clients/base/options.d.ts

@@ -82,6 +82,10 @@ export declare const baseOptionsSchema: {
       type: string;
       additionalProperties: boolean;
     };
+    ssl: {
+      type: string;
+      additionalProperties: boolean;
+    };
     tlsServerName: {
       oneOf: {
         type: string;

package/dist/typescript-4/dist/clients/consumer/consumer.d.ts

@@ -1,7 +1,6 @@
 import { type CallbackWithPromise } from "../../apis/callbacks";
 import { type FetchResponse } from "../../apis/consumer/fetch-v17";
-import { type ConnectionPool } from "../../network/connection-pool";
-import { Base, type BaseEvents, kFetchConnections } from "../base/base";
+import { Base, type BaseEvents } from "../base/base";
 import { MessagesStream } from "./messages-stream";
 import { TopicsMap } from "./topics-map";
 import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
@@ -20,11 +19,11 @@ export interface ConsumerEvents extends BaseEvents {
 export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ConsumerOptions<Key, Value, HeaderKey, HeaderValue>, ConsumerEvents> {
   #private;
   groupId: string;
+  groupInstanceId: string | null;
   generationId: number;
   memberId: string | null;
   topics: TopicsMap;
   assignments: GroupAssignment[] | null;
-  [kFetchConnections]: ConnectionPool;
   constructor(options: ConsumerOptions<Key, Value, HeaderKey, HeaderValue>);
   get streamsCount(): number;
   get lastHeartbeat(): Date | null;

package/dist/typescript-4/dist/clients/consumer/messages-stream.d.ts

@@ -1,8 +1,9 @@
 import { Readable } from 'node:stream';
 import { type CallbackWithPromise } from "../../apis/callbacks";
+import { type ConnectionPool } from "../../network/connection-pool";
 import { type Message } from "../../protocol/records";
 import { kAutocommit, kInstance, kRefreshOffsetsAndFetch } from "../../symbols";
-import { kInspect } from "../base/base";
+import { kConnections, kInspect } from "../base/base";
 import { type Consumer } from "./consumer";
 import { type CommitOptionsPartition, type ConsumeOptions } from "./types";
 export declare function noopDeserializer(data?: Buffer): Buffer | undefined;
@@ -10,6 +11,7 @@ export declare function defaultCorruptedMessageHandler(): boolean;
 export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends Readable {
   #private;
   [kInstance]: number;
+  [kConnections]: ConnectionPool;
   constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
   get consumer(): Consumer<Key, Value, HeaderKey, HeaderValue>;
   get offsetsToFetch(): Map<string, bigint>;

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -1,4 +1,8 @@
 export declare const groupOptionsProperties: {
+  groupInstanceId: {
+    type: string;
+    pattern: string;
+  };
   sessionTimeout: {
     type: string;
     minimum: number;
@@ -127,6 +131,10 @@ export declare const consumeOptionsProperties: {
 export declare const groupOptionsSchema: {
   type: string;
   properties: {
+    groupInstanceId: {
+      type: string;
+      pattern: string;
+    };
     sessionTimeout: {
       type: string;
       minimum: number;
@@ -233,6 +241,10 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    groupInstanceId: {
+      type: string;
+      pattern: string;
+    };
     sessionTimeout: {
       type: string;
       minimum: number;
@@ -384,6 +396,10 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    groupInstanceId: {
+      type: string;
+      pattern: string;
+    };
     sessionTimeout: {
       type: string;
       minimum: number;
@@ -495,6 +511,10 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    groupInstanceId: {
+      type: string;
+      pattern: string;
+    };
     sessionTimeout: {
       type: string;
       minimum: number;
@@ -588,6 +608,9 @@ export declare const fetchOptionsSchema: {
         required: string[];
       };
     };
+    connectionPool: {
+      type: string;
+    };
   };
   required: string[];
   additionalProperties: boolean;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -1,5 +1,6 @@
 import { type FetchRequestTopic } from "../../apis/consumer/fetch-v17";
 import { type GroupProtocols } from "../../apis/enumerations";
+import { type ConnectionPool } from "../../network/connection-pool";
 import { type KafkaRecord, type Message } from "../../protocol/records";
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
 import { type Deserializers } from "../serde";
@@ -46,6 +47,7 @@ export type MessagesStreamFallbackMode = keyof typeof MessagesStreamFallbackMode
 export type MessagesStreamFallbackModeValue = (typeof MessagesStreamFallbackModes)[keyof typeof MessagesStreamFallbackModes];
 export interface GroupOptions {
   groupProtocol?: typeof GroupProtocols.CLASSIC;
+  groupInstanceId?: string;
   sessionTimeout?: number;
   rebalanceTimeout?: number;
   heartbeatInterval?: number;
@@ -54,6 +56,7 @@ export interface GroupOptions {
 }
 export interface ConsumerGroupOptions {
   groupProtocol: typeof GroupProtocols.CONSUMER;
+  groupInstanceId?: string;
   groupRemoteAssignor?: string;
   rebalanceTimeout?: number;
 }
@@ -81,6 +84,7 @@ export type ConsumerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions &
 export type FetchOptions<Key, Value, HeaderKey, HeaderValue> = Pick<ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue>, 'minBytes' | 'maxBytes' | 'maxWaitTime' | 'isolationLevel'> & {
   node: number;
   topics: FetchRequestTopic[];
+  connectionPool?: ConnectionPool;
 };
 export interface CommitOptionsPartition extends TopicWithPartitionAndOffset {
   leaderEpoch: number;

package/dist/typescript-4/dist/network/connection.d.ts

@@ -4,8 +4,8 @@ import { type CallbackWithPromise } from "../apis/callbacks";
 import { type Callback, type ResponseParser } from "../apis/definitions";
 import { type SASLMechanismValue } from "../apis/enumerations";
 import { type SaslAuthenticateResponse, type SASLAuthenticationAPI } from "../apis/security/sasl-authenticate-v2";
-import { TypedEventEmitter, type TypedEvents } from "../events";
 import { TimeoutError } from "../errors";
+import { TypedEventEmitter, type TypedEvents } from "../events";
 import { Writer } from "../protocol/writer";
 export interface ConnectionEvents extends TypedEvents {
   connecting: () => void;
@@ -40,6 +40,7 @@ export interface ConnectionOptions {
   requestTimeout?: number;
   maxInflights?: number;
   tls?: TLSConnectionOptions;
+  ssl?: TLSConnectionOptions;
   tlsServerName?: string | boolean;
   sasl?: SASLOptions;
   ownerId?: number;
@@ -62,6 +63,7 @@ export declare const ConnectionStatuses: {
   readonly NONE: "none";
   readonly CONNECTING: "connecting";
   readonly AUTHENTICATING: "authenticating";
+  readonly REAUTHENTICATING: "reauthenticating";
   readonly CONNECTED: "connected";
   readonly CLOSED: "closed";
   readonly CLOSING: "closing";
@@ -89,4 +91,5 @@ export declare class Connection extends TypedEventEmitter<ConnectionEvents> {
   close(callback: CallbackWithPromise<void>): void;
   close(): Promise<void>;
   send<ReturnType>(apiKey: number, apiVersion: number, createPayload: () => Writer, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
+  reauthenticate(): void;
 }

package/dist/typescript-4/dist/protocol/records.d.ts

@@ -24,11 +24,22 @@ export interface MessageConsumerMetadata {
   generationId: number;
   memberId: string;
 }
+export interface MessageJSON<Key = unknown, Value = unknown, HeaderKey = unknown, HeaderValue = unknown> {
+  key: Key;
+  value: Value;
+  headers: Array<[HeaderKey, HeaderValue]>;
+  topic: string;
+  partition: number;
+  timestamp: string;
+  offset: string;
+  metadata: Record<string, unknown>;
+}
 export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
   headers: Map<HeaderKey, HeaderValue>;
   offset: bigint;
   metadata: Record<string, unknown>;
   commit(callback?: (error?: Error) => void): void | Promise<void>;
+  toJSON(): MessageJSON<Key, Value, HeaderKey, HeaderValue>;
 }
 export interface MessageRecord {
   key?: Buffer;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.25.0";
+export const version = "1.26.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.25.0",
+  "version": "1.26.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",