@platformatic/kafka 2.3.0 → 2.3.1

package/dist/clients/consumer/consumer.js

@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerLagChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { protocolErrors, UserError } from "../../errors.js";
+import { NetworkError, protocolErrors, UserError } from "../../errors.js";
 import { INT32_SIZE } from "../../protocol/definitions.js";
 import { Reader } from "../../protocol/reader.js";
 import { IS_CONTROL } from "../../protocol/records.js";
@@ -512,10 +512,64 @@ export class Consumer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        const session = this.#fetchSessions.get(node);
+                        let session = this.#fetchSessions.get(node);
+                        if (session && session.pool !== pool) {
+                            session = undefined;
+                        }
                         const sessionId = session?.id ?? 0;
                         const sessionEpoch = session?.epoch ?? 0;
-                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, options.topics, [], this.#clientRack, (error, result) => {
+                        // The desired state of the session: options.topics always lists all the wanted partitions
+                        const desired = new Map();
+                        for (const topic of options.topics) {
+                            let partitions = desired.get(topic.topicId);
+                            if (!partitions) {
+                                partitions = new Map();
+                                desired.set(topic.topicId, partitions);
+                            }
+                            for (const partition of topic.partitions) {
+                                // Copy the partition since the data is mutated in the error paths
+                                partitions.set(partition.partition, { ...partition });
+                            }
+                        }
+                        let requestTopics = options.topics;
+                        const forgottenTopicsData = [];
+                        if (session) {
+                            // Only send the partitions which are new or whose state changed: the broker
+                            // keeps serving the unchanged ones from the session (KIP-227)
+                            requestTopics = [];
+                            for (const [topicId, partitions] of desired) {
+                                const cached = session.partitions.get(topicId);
+                                const changed = [];
+                                for (const partition of partitions.values()) {
+                                    const previous = cached?.get(partition.partition);
+                                    if (!previous ||
+                                        previous.fetchOffset !== partition.fetchOffset ||
+                                        previous.currentLeaderEpoch !== partition.currentLeaderEpoch ||
+                                        previous.lastFetchedEpoch !== partition.lastFetchedEpoch ||
+                                        previous.partitionMaxBytes !== partition.partitionMaxBytes) {
+                                        changed.push(partition);
+                                    }
+                                }
+                                if (changed.length > 0) {
+                                    requestTopics.push({ topicId, partitions: changed });
+                                }
+                            }
+                            // Ask the broker to remove from the session the partitions which are no
+                            // longer wanted, otherwise it keeps serving them forever
+                            for (const [topicId, partitions] of session.partitions) {
+                                const stillDesired = desired.get(topicId);
+                                const forgotten = [];
+                                for (const partition of partitions.keys()) {
+                                    if (!stillDesired?.has(partition)) {
+                                        forgotten.push(partition);
+                                    }
+                                }
+                                if (forgotten.length > 0) {
+                                    forgottenTopicsData.push({ topic: topicId, partitions: forgotten });
+                                }
+                            }
+                        }
+                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, requestTopics, forgottenTopicsData, this.#clientRack, (error, result) => {
                             if (error) {
                                 this.#clearPreferredReadReplicas(options.topics, topicIds);
                                 const genericError = error;
@@ -537,7 +591,7 @@ export class Consumer extends Base {
                             }
                             else {
                                 this.#updatePreferredReadReplicas(metadata, topicIds, result);
-                                this.#updateFetchSession(node, sessionEpoch, result.sessionId);
+                                this.#updateFetchSession(node, pool, sessionEpoch, result.sessionId, desired);
                             }
                             retryCallback(error, result);
                         });
@@ -556,12 +610,19 @@ export class Consumer extends Base {
             callback(error, response);
         }, 0);
     }
-    #updateFetchSession(node, sentEpoch, responseSessionId) {
+    #updateFetchSession(node, pool, sentEpoch, responseSessionId, desired) {
         if (!responseSessionId) {
             this.#fetchSessions.delete(node);
             return;
         }
-        this.#fetchSessions.set(node, { id: responseSessionId, epoch: nextFetchEpoch(sentEpoch) });
+        // After a successful fetch the broker session contains exactly the desired partitions:
+        // unchanged ones already matched it, changed ones were sent and removed ones were forgotten
+        this.#fetchSessions.set(node, {
+            id: responseSessionId,
+            epoch: nextFetchEpoch(sentEpoch),
+            pool,
+            partitions: desired
+        });
     }
     #commit(options, callback, rejoinAttempts = 0) {
         this.#performGroupOperation('commit', (connection, groupCallback) => {
@@ -1230,11 +1291,17 @@ export class Consumer extends Base {
                         topicSubscriptions.push(subscription);
                     }
                 }
-                this[kMetadata]({ topics: Array.from(topicsSubscriptions.keys()) }, (error, metadata) => {
+                const subscribedTopics = Array.from(topicsSubscriptions.keys());
+                this[kMetadata]({ topics: subscribedTopics }, (error, metadata) => {
                     if (error) {
                         callback(this.#handleError(error));
                         return;
                     }
+                    const missingTopic = subscribedTopics.find(topic => !metadata.topics.has(topic));
+                    if (missingTopic) {
+                        callback(this.#handleError(new NetworkError(`Cannot compute group assignments: metadata for subscribed topic "${missingTopic}" is not available in the current cluster metadata.`, { canRetry: true, hasStaleMetadata: true })));
+                        return;
+                    }
                     this.#performSyncGroup(partitionsAssigner, this.#createAssignments(partitionsAssigner, metadata), callback);
                 });
                 return;

package/dist/clients/consumer/messages-stream.js

@@ -553,6 +553,11 @@ export class MessagesStream extends Readable {
             const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
                 const key = partitionKey(topic, partition);
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - their data is stale and must not rewind offsets
+                if (!requestedOffsets.has(key)) {
+                    continue;
+                }
                 if (!recordsBatches) {
                     continue;
                 }
@@ -860,7 +865,13 @@ export class MessagesStream extends Readable {
         const requests = [];
         // Create the pre-deserialization requests
         for (const topicResponse of response.responses) {
+            const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - do not run hooks on their stale records
+                if (!requestedOffsets.has(partitionKey(topic, partition))) {
+                    continue;
+                }
                 /* c8 ignore next 3 - Hard to test */
                 if (!recordsBatches) {
                     continue;
@@ -872,7 +883,7 @@ export class MessagesStream extends Readable {
                         continue;
                     }
                     for (const message of batch.records) {
-                        message.topic = topicIds.get(topicResponse.topicId);
+                        message.topic = topic;
                         message.partition = partition;
                         requests.push([message.key, 'key', message]);
                         requests.push([message.value, 'value', message]);

package/dist/clients/producer/producer.js

@@ -342,7 +342,7 @@ export class Producer extends Base {
         }
         const transactionalId = this.#transaction.id;
         this[kPerformDeduplicated]('findCoordinator', deduplicateCallback => {
-            this[kPerformWithRetry]('findCoordinator', retryCallback => {
+            this[kPerformWithRetry]('findCoordinator', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
@@ -355,7 +355,7 @@ export class Producer extends Base {
                         }
                         api(connection, FindCoordinatorKeyTypes.TRANSACTION, [transactionalId], retryCallback);
                     });
-                });
+                }, attempt);
             }, (error, response) => {
                 if (error) {
                     deduplicateCallback(error);
@@ -531,11 +531,8 @@ export class Producer extends Base {
     #initIdempotentProducer(options, callback) {
         const transactionalId = this.#transaction?.id;
         this[kPerformDeduplicated]('initProducerId', deduplicateCallback => {
-            this[kPerformWithRetry]('initProducerId', retryCallback => {
-                const connector = transactionalId
-                    ? this.#getCoordinatorConnection.bind(this)
-                    : this[kGetBootstrapConnection].bind(this);
-                connector((error, connection) => {
+            this[kPerformWithRetry]('initProducerId', (retryCallback, attempt) => {
+                const onConnection = (error, connection) => {
                     if (error) {
                         retryCallback(error);
                         return;
@@ -549,7 +546,16 @@ export class Producer extends Base {
                             retryCallback(this.#handleError(error), response);
                         });
                     });
-                });
+                };
+                // The transactional path fails over inside #getCoordinatorConnection (via
+                // #findCoordinator -> kGetBootstrapConnection). The idempotent path must thread
+                // `attempt` so kGetBootstrapConnection rotates past a reachable-but-hung broker.
+                if (transactionalId) {
+                    this.#getCoordinatorConnection(onConnection);
+                }
+                else {
+                    this[kGetBootstrapConnection](onConnection, attempt);
+                }
             }, (error, response) => {
                 if (error) {
                     this.#handleFencingError(error);

package/dist/network/connection-pool.js

@@ -1,6 +1,6 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../apis/callbacks.js";
 import { connectionsPoolGetsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
-import { MultipleErrors } from "../errors.js";
+import { MultipleErrors, NetworkError } from "../errors.js";
 import { TypedEventEmitter } from "../events.js";
 import { Connection, ConnectionStatuses } from "./connection.js";
 let currentInstance = 0;
@@ -95,6 +95,14 @@ export class ConnectionPool extends TypedEventEmitter {
             callback(new Error('Connection pool is closed.'));
             return;
         }
+        if (broker == null || broker.host == null || broker.port == null) {
+            callback(new NetworkError('Cannot get a connection: the broker is not available in the current cluster metadata.', {
+                canRetry: true,
+                hasStaleMetadata: true,
+                broker
+            }));
+            return;
+        }
         const key = `${broker.host}:${broker.port}`;
         const existing = this.#connections.get(key);
         if (existing) {

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.3.0";
+export const version = "2.3.1";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.3.0",
+  "version": "2.3.1",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",