@platformatic/kafka 2.3.1 → 2.4.0-alpha.1

package/dist/clients/consumer/consumer.js

@@ -395,6 +395,10 @@ export class Consumer extends Base {
         options.rebalanceTimeout ??= this[kOptions].rebalanceTimeout;
         options.heartbeatInterval ??= this[kOptions].heartbeatInterval;
         options.protocols ??= this[kOptions].protocols;
+        const protocolsMetadata = this[kOptions].protocolsMetadata;
+        if (!options.protocolsMetadata && protocolsMetadata) {
+            options.protocolsMetadata = protocolsMetadata;
+        }
         this.#validateGroupOptions(options);
         this.#membershipActive = true;
         this.#joinGroup(options, callback);
@@ -1146,45 +1150,20 @@ export class Consumer extends Base {
             return;
         }
         this.#cancelHeartbeat();
-        const protocols = [];
-        for (const protocol of options.protocols) {
-            protocols.push({
-                name: protocol.name,
-                metadata: this.#encodeProtocolSubscriptionMetadata(protocol, this.topics.current)
-            });
-        }
-        this.#performDeduplicateGroupOperaton('joinGroup', (connection, groupCallback) => {
-            this[kGetApi]('JoinGroup', (error, api) => {
-                if (error) {
-                    groupCallback(error);
-                    return;
-                }
-                api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', this.groupInstanceId, 'consumer', protocols, '', groupCallback);
-            });
-        }, (error, response) => {
-            if (!this.#membershipActive) {
-                callback(null);
-                return;
-            }
+        this.#createJoinGroupProtocols(options, (error, protocols) => {
             if (error) {
-                if (this.#getRejoinError(error)) {
-                    this.#performJoinGroup(options, callback);
-                    return;
-                }
                 callback(error);
                 return;
             }
-            // This is for Azure Event Hubs compatibility, which does not respond with an error on the first join
-            this.memberId = response.memberId;
-            this.generationId = response.generationId;
-            this.#isLeader = response.leader === this.memberId;
-            this.#protocol = response.protocolName;
-            this.#members = new Map();
-            for (const member of response.members) {
-                this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
-            }
-            // Send a syncGroup request
-            this.#syncGroup(options.partitionAssigner, (error, response) => {
+            this.#performDeduplicateGroupOperaton('joinGroup', (connection, groupCallback) => {
+                this[kGetApi]('JoinGroup', (error, api) => {
+                    if (error) {
+                        groupCallback(error);
+                        return;
+                    }
+                    api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', this.groupInstanceId, 'consumer', protocols, '', groupCallback);
+                });
+            }, (error, response) => {
                 if (!this.#membershipActive) {
                     callback(null);
                     return;
@@ -1197,20 +1176,44 @@ export class Consumer extends Base {
                     callback(error);
                     return;
                 }
-                this.assignments = response;
-                this.#syncPreferredReadReplicas();
-                this.#cancelHeartbeat();
-                this.#heartbeatInterval = setTimeout(() => {
-                    this.#heartbeat(options);
-                }, options.heartbeatInterval);
-                this.emitWithDebug('consumer', 'group:join', {
-                    groupId: this.groupId,
-                    memberId: this.memberId,
-                    generationId: this.generationId,
-                    isLeader: this.#isLeader,
-                    assignments: this.assignments
+                // This is for Azure Event Hubs compatibility, which does not respond with an error on the first join
+                this.memberId = response.memberId;
+                this.generationId = response.generationId;
+                this.#isLeader = response.leader === this.memberId;
+                this.#protocol = response.protocolName;
+                this.#members = new Map();
+                for (const member of response.members) {
+                    this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
+                }
+                // Send a syncGroup request
+                this.#syncGroup(options.partitionAssigner, (error, response) => {
+                    if (!this.#membershipActive) {
+                        callback(null);
+                        return;
+                    }
+                    if (error) {
+                        if (this.#getRejoinError(error)) {
+                            this.#performJoinGroup(options, callback);
+                            return;
+                        }
+                        callback(error);
+                        return;
+                    }
+                    this.assignments = response;
+                    this.#syncPreferredReadReplicas();
+                    this.#cancelHeartbeat();
+                    this.#heartbeatInterval = setTimeout(() => {
+                        this.#heartbeat(options);
+                    }, options.heartbeatInterval);
+                    this.emitWithDebug('consumer', 'group:join', {
+                        groupId: this.groupId,
+                        memberId: this.memberId,
+                        generationId: this.generationId,
+                        isLeader: this.#isLeader,
+                        assignments: this.assignments
+                    });
+                    callback(null, this.memberId);
                 });
-                callback(null, this.memberId);
             });
         });
     }
@@ -1370,6 +1373,61 @@ export class Consumer extends Base {
             throw new UserError(this[kFormatValidationErrors](validator, '/options'));
         }
     }
+    #createJoinGroupProtocols(options, callback) {
+        if (typeof options.protocolsMetadata !== 'function') {
+            callback(null, options.protocols.map(protocol => ({
+                name: protocol.name,
+                metadata: this.#encodeProtocolSubscriptionMetadata(protocol, this.topics.current)
+            })));
+            return;
+        }
+        this[kMetadata]({ topics: this.topics.current, forceUpdate: true }, (error, metadata) => {
+            if (error) {
+                callback(this.#handleError(error));
+                return;
+            }
+            this.#resolveProtocolsMetadata(options.protocolsMetadata, options.protocols, this.topics.current, metadata, (error, userData) => {
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                else if (!Buffer.isBuffer(userData)) {
+                    callback(new UserError('protocolsMetadata must resolve to a Buffer.'));
+                    return;
+                }
+                callback(null, options.protocols.map(protocol => ({
+                    name: protocol.name,
+                    metadata: this.#encodeProtocolSubscriptionMetadata({ ...protocol, metadata: userData }, this.topics.current)
+                })));
+            });
+        });
+    }
+    #resolveProtocolsMetadata(protocolsMetadata, protocols, topics, metadata, callback) {
+        let settled = false;
+        function resolveCallback(error, userData) {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            callback(error, userData);
+        }
+        let result;
+        try {
+            result = protocolsMetadata(protocols, topics, metadata, resolveCallback);
+            if (typeof result !== 'undefined') {
+                if (typeof result.then === 'function') {
+                    ;
+                    result.then(userData => process.nextTick(resolveCallback, null, userData), error => process.nextTick(resolveCallback, error));
+                }
+                else {
+                    resolveCallback(null, result);
+                }
+            }
+        }
+        catch (error) {
+            resolveCallback(error);
+        }
+    }
     /*
       The following two methods follow:
       https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json

package/dist/clients/consumer/options.d.ts

@@ -53,6 +53,9 @@ export declare const groupOptionsProperties: {
             };
         };
     };
+    protocolsMetadata: {
+        function: boolean;
+    };
     partitionAssigner: {
         function: boolean;
     };
@@ -195,6 +198,9 @@ export declare const groupOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -315,6 +321,9 @@ export declare const consumeOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -481,6 +490,9 @@ export declare const consumerOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -608,6 +620,9 @@ export declare const fetchOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };

package/dist/clients/consumer/options.js

@@ -25,6 +25,7 @@ export const groupOptionsProperties = {
             }
         }
     },
+    protocolsMetadata: { function: true },
     partitionAssigner: { function: true }
 };
 export const groupOptionsAdditionalValidations = {

package/dist/clients/consumer/types.d.ts

@@ -1,3 +1,4 @@
+import { type Callback } from '../../apis/definitions.ts';
 import { type FetchRequestTopic } from '../../apis/consumer/fetch-v17.ts';
 import { type GroupProtocols } from '../../apis/enumerations.ts';
 import { type ConnectionPool } from '../../network/connection-pool.ts';
@@ -33,6 +34,7 @@ export interface PreferredReadReplica {
 }
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
     readonly LATEST: "latest";
     readonly EARLIEST: "earliest";
@@ -57,6 +59,7 @@ export interface GroupOptions {
     rebalanceTimeout?: number;
     heartbeatInterval?: number;
     protocols?: GroupProtocolSubscription[];
+    protocolsMetadata?: GroupProtocolsMetadataCallback;
     partitionAssigner?: GroupPartitionsAssigner;
     assignmentUserData?: Buffer;
 }

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -53,6 +53,9 @@ export declare const groupOptionsProperties: {
       };
     };
   };
+  protocolsMetadata: {
+    function: boolean;
+  };
   partitionAssigner: {
     function: boolean;
   };
@@ -195,6 +198,9 @@ export declare const groupOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -315,6 +321,9 @@ export declare const consumeOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -481,6 +490,9 @@ export declare const consumerOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -608,6 +620,9 @@ export declare const fetchOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -1,3 +1,4 @@
+import { type Callback } from "../../apis/definitions";
 import { type FetchRequestTopic } from "../../apis/consumer/fetch-v17";
 import { type GroupProtocols } from "../../apis/enumerations";
 import { type ConnectionPool } from "../../network/connection-pool";
@@ -33,6 +34,7 @@ export interface PreferredReadReplica {
 }
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
   readonly LATEST: "latest";
   readonly EARLIEST: "earliest";
@@ -57,6 +59,7 @@ export interface GroupOptions {
   rebalanceTimeout?: number;
   heartbeatInterval?: number;
   protocols?: GroupProtocolSubscription[];
+  protocolsMetadata?: GroupProtocolsMetadataCallback;
   partitionAssigner?: GroupPartitionsAssigner;
   assignmentUserData?: Buffer;
 }

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.3.1";
+export const version = "2.4.0-alpha.1";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.3.1",
+  "version": "2.4.0-alpha.1",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",