@platformatic/kafka 2.3.0 → 2.4.0-alpha.1

package/dist/clients/consumer/consumer.js

@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerLagChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { protocolErrors, UserError } from "../../errors.js";
+import { NetworkError, protocolErrors, UserError } from "../../errors.js";
 import { INT32_SIZE } from "../../protocol/definitions.js";
 import { Reader } from "../../protocol/reader.js";
 import { IS_CONTROL } from "../../protocol/records.js";
@@ -395,6 +395,10 @@ export class Consumer extends Base {
         options.rebalanceTimeout ??= this[kOptions].rebalanceTimeout;
         options.heartbeatInterval ??= this[kOptions].heartbeatInterval;
         options.protocols ??= this[kOptions].protocols;
+        const protocolsMetadata = this[kOptions].protocolsMetadata;
+        if (!options.protocolsMetadata && protocolsMetadata) {
+            options.protocolsMetadata = protocolsMetadata;
+        }
         this.#validateGroupOptions(options);
         this.#membershipActive = true;
         this.#joinGroup(options, callback);
@@ -512,10 +516,64 @@ export class Consumer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        const session = this.#fetchSessions.get(node);
+                        let session = this.#fetchSessions.get(node);
+                        if (session && session.pool !== pool) {
+                            session = undefined;
+                        }
                         const sessionId = session?.id ?? 0;
                         const sessionEpoch = session?.epoch ?? 0;
-                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, options.topics, [], this.#clientRack, (error, result) => {
+                        // The desired state of the session: options.topics always lists all the wanted partitions
+                        const desired = new Map();
+                        for (const topic of options.topics) {
+                            let partitions = desired.get(topic.topicId);
+                            if (!partitions) {
+                                partitions = new Map();
+                                desired.set(topic.topicId, partitions);
+                            }
+                            for (const partition of topic.partitions) {
+                                // Copy the partition since the data is mutated in the error paths
+                                partitions.set(partition.partition, { ...partition });
+                            }
+                        }
+                        let requestTopics = options.topics;
+                        const forgottenTopicsData = [];
+                        if (session) {
+                            // Only send the partitions which are new or whose state changed: the broker
+                            // keeps serving the unchanged ones from the session (KIP-227)
+                            requestTopics = [];
+                            for (const [topicId, partitions] of desired) {
+                                const cached = session.partitions.get(topicId);
+                                const changed = [];
+                                for (const partition of partitions.values()) {
+                                    const previous = cached?.get(partition.partition);
+                                    if (!previous ||
+                                        previous.fetchOffset !== partition.fetchOffset ||
+                                        previous.currentLeaderEpoch !== partition.currentLeaderEpoch ||
+                                        previous.lastFetchedEpoch !== partition.lastFetchedEpoch ||
+                                        previous.partitionMaxBytes !== partition.partitionMaxBytes) {
+                                        changed.push(partition);
+                                    }
+                                }
+                                if (changed.length > 0) {
+                                    requestTopics.push({ topicId, partitions: changed });
+                                }
+                            }
+                            // Ask the broker to remove from the session the partitions which are no
+                            // longer wanted, otherwise it keeps serving them forever
+                            for (const [topicId, partitions] of session.partitions) {
+                                const stillDesired = desired.get(topicId);
+                                const forgotten = [];
+                                for (const partition of partitions.keys()) {
+                                    if (!stillDesired?.has(partition)) {
+                                        forgotten.push(partition);
+                                    }
+                                }
+                                if (forgotten.length > 0) {
+                                    forgottenTopicsData.push({ topic: topicId, partitions: forgotten });
+                                }
+                            }
+                        }
+                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, requestTopics, forgottenTopicsData, this.#clientRack, (error, result) => {
                             if (error) {
                                 this.#clearPreferredReadReplicas(options.topics, topicIds);
                                 const genericError = error;
@@ -537,7 +595,7 @@ export class Consumer extends Base {
                             }
                             else {
                                 this.#updatePreferredReadReplicas(metadata, topicIds, result);
-                                this.#updateFetchSession(node, sessionEpoch, result.sessionId);
+                                this.#updateFetchSession(node, pool, sessionEpoch, result.sessionId, desired);
                             }
                             retryCallback(error, result);
                         });
@@ -556,12 +614,19 @@ export class Consumer extends Base {
             callback(error, response);
         }, 0);
     }
-    #updateFetchSession(node, sentEpoch, responseSessionId) {
+    #updateFetchSession(node, pool, sentEpoch, responseSessionId, desired) {
         if (!responseSessionId) {
             this.#fetchSessions.delete(node);
             return;
         }
-        this.#fetchSessions.set(node, { id: responseSessionId, epoch: nextFetchEpoch(sentEpoch) });
+        // After a successful fetch the broker session contains exactly the desired partitions:
+        // unchanged ones already matched it, changed ones were sent and removed ones were forgotten
+        this.#fetchSessions.set(node, {
+            id: responseSessionId,
+            epoch: nextFetchEpoch(sentEpoch),
+            pool,
+            partitions: desired
+        });
     }
     #commit(options, callback, rejoinAttempts = 0) {
         this.#performGroupOperation('commit', (connection, groupCallback) => {
@@ -1085,45 +1150,20 @@ export class Consumer extends Base {
             return;
         }
         this.#cancelHeartbeat();
-        const protocols = [];
-        for (const protocol of options.protocols) {
-            protocols.push({
-                name: protocol.name,
-                metadata: this.#encodeProtocolSubscriptionMetadata(protocol, this.topics.current)
-            });
-        }
-        this.#performDeduplicateGroupOperaton('joinGroup', (connection, groupCallback) => {
-            this[kGetApi]('JoinGroup', (error, api) => {
-                if (error) {
-                    groupCallback(error);
-                    return;
-                }
-                api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', this.groupInstanceId, 'consumer', protocols, '', groupCallback);
-            });
-        }, (error, response) => {
-            if (!this.#membershipActive) {
-                callback(null);
-                return;
-            }
+        this.#createJoinGroupProtocols(options, (error, protocols) => {
             if (error) {
-                if (this.#getRejoinError(error)) {
-                    this.#performJoinGroup(options, callback);
-                    return;
-                }
                 callback(error);
                 return;
             }
-            // This is for Azure Event Hubs compatibility, which does not respond with an error on the first join
-            this.memberId = response.memberId;
-            this.generationId = response.generationId;
-            this.#isLeader = response.leader === this.memberId;
-            this.#protocol = response.protocolName;
-            this.#members = new Map();
-            for (const member of response.members) {
-                this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
-            }
-            // Send a syncGroup request
-            this.#syncGroup(options.partitionAssigner, (error, response) => {
+            this.#performDeduplicateGroupOperaton('joinGroup', (connection, groupCallback) => {
+                this[kGetApi]('JoinGroup', (error, api) => {
+                    if (error) {
+                        groupCallback(error);
+                        return;
+                    }
+                    api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', this.groupInstanceId, 'consumer', protocols, '', groupCallback);
+                });
+            }, (error, response) => {
                 if (!this.#membershipActive) {
                     callback(null);
                     return;
@@ -1136,20 +1176,44 @@ export class Consumer extends Base {
                     callback(error);
                     return;
                 }
-                this.assignments = response;
-                this.#syncPreferredReadReplicas();
-                this.#cancelHeartbeat();
-                this.#heartbeatInterval = setTimeout(() => {
-                    this.#heartbeat(options);
-                }, options.heartbeatInterval);
-                this.emitWithDebug('consumer', 'group:join', {
-                    groupId: this.groupId,
-                    memberId: this.memberId,
-                    generationId: this.generationId,
-                    isLeader: this.#isLeader,
-                    assignments: this.assignments
+                // This is for Azure Event Hubs compatibility, which does not respond with an error on the first join
+                this.memberId = response.memberId;
+                this.generationId = response.generationId;
+                this.#isLeader = response.leader === this.memberId;
+                this.#protocol = response.protocolName;
+                this.#members = new Map();
+                for (const member of response.members) {
+                    this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
+                }
+                // Send a syncGroup request
+                this.#syncGroup(options.partitionAssigner, (error, response) => {
+                    if (!this.#membershipActive) {
+                        callback(null);
+                        return;
+                    }
+                    if (error) {
+                        if (this.#getRejoinError(error)) {
+                            this.#performJoinGroup(options, callback);
+                            return;
+                        }
+                        callback(error);
+                        return;
+                    }
+                    this.assignments = response;
+                    this.#syncPreferredReadReplicas();
+                    this.#cancelHeartbeat();
+                    this.#heartbeatInterval = setTimeout(() => {
+                        this.#heartbeat(options);
+                    }, options.heartbeatInterval);
+                    this.emitWithDebug('consumer', 'group:join', {
+                        groupId: this.groupId,
+                        memberId: this.memberId,
+                        generationId: this.generationId,
+                        isLeader: this.#isLeader,
+                        assignments: this.assignments
+                    });
+                    callback(null, this.memberId);
                 });
-                callback(null, this.memberId);
             });
         });
     }
@@ -1230,11 +1294,17 @@ export class Consumer extends Base {
                         topicSubscriptions.push(subscription);
                     }
                 }
-                this[kMetadata]({ topics: Array.from(topicsSubscriptions.keys()) }, (error, metadata) => {
+                const subscribedTopics = Array.from(topicsSubscriptions.keys());
+                this[kMetadata]({ topics: subscribedTopics }, (error, metadata) => {
                     if (error) {
                         callback(this.#handleError(error));
                         return;
                     }
+                    const missingTopic = subscribedTopics.find(topic => !metadata.topics.has(topic));
+                    if (missingTopic) {
+                        callback(this.#handleError(new NetworkError(`Cannot compute group assignments: metadata for subscribed topic "${missingTopic}" is not available in the current cluster metadata.`, { canRetry: true, hasStaleMetadata: true })));
+                        return;
+                    }
                     this.#performSyncGroup(partitionsAssigner, this.#createAssignments(partitionsAssigner, metadata), callback);
                 });
                 return;
@@ -1303,6 +1373,61 @@ export class Consumer extends Base {
             throw new UserError(this[kFormatValidationErrors](validator, '/options'));
         }
     }
+    #createJoinGroupProtocols(options, callback) {
+        if (typeof options.protocolsMetadata !== 'function') {
+            callback(null, options.protocols.map(protocol => ({
+                name: protocol.name,
+                metadata: this.#encodeProtocolSubscriptionMetadata(protocol, this.topics.current)
+            })));
+            return;
+        }
+        this[kMetadata]({ topics: this.topics.current, forceUpdate: true }, (error, metadata) => {
+            if (error) {
+                callback(this.#handleError(error));
+                return;
+            }
+            this.#resolveProtocolsMetadata(options.protocolsMetadata, options.protocols, this.topics.current, metadata, (error, userData) => {
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                else if (!Buffer.isBuffer(userData)) {
+                    callback(new UserError('protocolsMetadata must resolve to a Buffer.'));
+                    return;
+                }
+                callback(null, options.protocols.map(protocol => ({
+                    name: protocol.name,
+                    metadata: this.#encodeProtocolSubscriptionMetadata({ ...protocol, metadata: userData }, this.topics.current)
+                })));
+            });
+        });
+    }
+    #resolveProtocolsMetadata(protocolsMetadata, protocols, topics, metadata, callback) {
+        let settled = false;
+        function resolveCallback(error, userData) {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            callback(error, userData);
+        }
+        let result;
+        try {
+            result = protocolsMetadata(protocols, topics, metadata, resolveCallback);
+            if (typeof result !== 'undefined') {
+                if (typeof result.then === 'function') {
+                    ;
+                    result.then(userData => process.nextTick(resolveCallback, null, userData), error => process.nextTick(resolveCallback, error));
+                }
+                else {
+                    resolveCallback(null, result);
+                }
+            }
+        }
+        catch (error) {
+            resolveCallback(error);
+        }
+    }
     /*
       The following two methods follow:
       https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json

package/dist/clients/consumer/messages-stream.js

@@ -553,6 +553,11 @@ export class MessagesStream extends Readable {
             const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
                 const key = partitionKey(topic, partition);
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - their data is stale and must not rewind offsets
+                if (!requestedOffsets.has(key)) {
+                    continue;
+                }
                 if (!recordsBatches) {
                     continue;
                 }
@@ -860,7 +865,13 @@ export class MessagesStream extends Readable {
         const requests = [];
         // Create the pre-deserialization requests
         for (const topicResponse of response.responses) {
+            const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - do not run hooks on their stale records
+                if (!requestedOffsets.has(partitionKey(topic, partition))) {
+                    continue;
+                }
                 /* c8 ignore next 3 - Hard to test */
                 if (!recordsBatches) {
                     continue;
@@ -872,7 +883,7 @@ export class MessagesStream extends Readable {
                         continue;
                     }
                     for (const message of batch.records) {
-                        message.topic = topicIds.get(topicResponse.topicId);
+                        message.topic = topic;
                         message.partition = partition;
                         requests.push([message.key, 'key', message]);
                         requests.push([message.value, 'value', message]);

package/dist/clients/consumer/options.d.ts

@@ -53,6 +53,9 @@ export declare const groupOptionsProperties: {
             };
         };
     };
+    protocolsMetadata: {
+        function: boolean;
+    };
     partitionAssigner: {
         function: boolean;
     };
@@ -195,6 +198,9 @@ export declare const groupOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -315,6 +321,9 @@ export declare const consumeOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -481,6 +490,9 @@ export declare const consumerOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -608,6 +620,9 @@ export declare const fetchOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };

package/dist/clients/consumer/options.js

@@ -25,6 +25,7 @@ export const groupOptionsProperties = {
             }
         }
     },
+    protocolsMetadata: { function: true },
     partitionAssigner: { function: true }
 };
 export const groupOptionsAdditionalValidations = {

package/dist/clients/consumer/types.d.ts

@@ -1,3 +1,4 @@
+import { type Callback } from '../../apis/definitions.ts';
 import { type FetchRequestTopic } from '../../apis/consumer/fetch-v17.ts';
 import { type GroupProtocols } from '../../apis/enumerations.ts';
 import { type ConnectionPool } from '../../network/connection-pool.ts';
@@ -33,6 +34,7 @@ export interface PreferredReadReplica {
 }
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
     readonly LATEST: "latest";
     readonly EARLIEST: "earliest";
@@ -57,6 +59,7 @@ export interface GroupOptions {
     rebalanceTimeout?: number;
     heartbeatInterval?: number;
     protocols?: GroupProtocolSubscription[];
+    protocolsMetadata?: GroupProtocolsMetadataCallback;
     partitionAssigner?: GroupPartitionsAssigner;
     assignmentUserData?: Buffer;
 }

package/dist/clients/producer/producer.js

@@ -342,7 +342,7 @@ export class Producer extends Base {
         }
         const transactionalId = this.#transaction.id;
         this[kPerformDeduplicated]('findCoordinator', deduplicateCallback => {
-            this[kPerformWithRetry]('findCoordinator', retryCallback => {
+            this[kPerformWithRetry]('findCoordinator', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
@@ -355,7 +355,7 @@ export class Producer extends Base {
                         }
                         api(connection, FindCoordinatorKeyTypes.TRANSACTION, [transactionalId], retryCallback);
                     });
-                });
+                }, attempt);
             }, (error, response) => {
                 if (error) {
                     deduplicateCallback(error);
@@ -531,11 +531,8 @@ export class Producer extends Base {
     #initIdempotentProducer(options, callback) {
         const transactionalId = this.#transaction?.id;
         this[kPerformDeduplicated]('initProducerId', deduplicateCallback => {
-            this[kPerformWithRetry]('initProducerId', retryCallback => {
-                const connector = transactionalId
-                    ? this.#getCoordinatorConnection.bind(this)
-                    : this[kGetBootstrapConnection].bind(this);
-                connector((error, connection) => {
+            this[kPerformWithRetry]('initProducerId', (retryCallback, attempt) => {
+                const onConnection = (error, connection) => {
                     if (error) {
                         retryCallback(error);
                         return;
@@ -549,7 +546,16 @@ export class Producer extends Base {
                             retryCallback(this.#handleError(error), response);
                         });
                     });
-                });
+                };
+                // The transactional path fails over inside #getCoordinatorConnection (via
+                // #findCoordinator -> kGetBootstrapConnection). The idempotent path must thread
+                // `attempt` so kGetBootstrapConnection rotates past a reachable-but-hung broker.
+                if (transactionalId) {
+                    this.#getCoordinatorConnection(onConnection);
+                }
+                else {
+                    this[kGetBootstrapConnection](onConnection, attempt);
+                }
             }, (error, response) => {
                 if (error) {
                     this.#handleFencingError(error);

package/dist/network/connection-pool.js

@@ -1,6 +1,6 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../apis/callbacks.js";
 import { connectionsPoolGetsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
-import { MultipleErrors } from "../errors.js";
+import { MultipleErrors, NetworkError } from "../errors.js";
 import { TypedEventEmitter } from "../events.js";
 import { Connection, ConnectionStatuses } from "./connection.js";
 let currentInstance = 0;
@@ -95,6 +95,14 @@ export class ConnectionPool extends TypedEventEmitter {
             callback(new Error('Connection pool is closed.'));
             return;
         }
+        if (broker == null || broker.host == null || broker.port == null) {
+            callback(new NetworkError('Cannot get a connection: the broker is not available in the current cluster metadata.', {
+                canRetry: true,
+                hasStaleMetadata: true,
+                broker
+            }));
+            return;
+        }
         const key = `${broker.host}:${broker.port}`;
         const existing = this.#connections.get(key);
         if (existing) {

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -53,6 +53,9 @@ export declare const groupOptionsProperties: {
       };
     };
   };
+  protocolsMetadata: {
+    function: boolean;
+  };
   partitionAssigner: {
     function: boolean;
   };
@@ -195,6 +198,9 @@ export declare const groupOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -315,6 +321,9 @@ export declare const consumeOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -481,6 +490,9 @@ export declare const consumerOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -608,6 +620,9 @@ export declare const fetchOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -1,3 +1,4 @@
+import { type Callback } from "../../apis/definitions";
 import { type FetchRequestTopic } from "../../apis/consumer/fetch-v17";
 import { type GroupProtocols } from "../../apis/enumerations";
 import { type ConnectionPool } from "../../network/connection-pool";
@@ -33,6 +34,7 @@ export interface PreferredReadReplica {
 }
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
   readonly LATEST: "latest";
   readonly EARLIEST: "earliest";
@@ -57,6 +59,7 @@ export interface GroupOptions {
   rebalanceTimeout?: number;
   heartbeatInterval?: number;
   protocols?: GroupProtocolSubscription[];
+  protocolsMetadata?: GroupProtocolsMetadataCallback;
   partitionAssigner?: GroupPartitionsAssigner;
   assignmentUserData?: Buffer;
 }

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.3.0";
+export const version = "2.4.0-alpha.1";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.3.0",
+  "version": "2.4.0-alpha.1",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",