@platformatic/kafka 1.0.0 → 1.2.0

package/dist/clients/consumer/consumer.js

@@ -1,17 +1,9 @@
-import { api as fetchV17 } from "../../apis/consumer/fetch.js";
-import { api as heartbeatV4 } from "../../apis/consumer/heartbeat.js";
-import { api as joinGroupV9 } from "../../apis/consumer/join-group.js";
-import { api as leaveGroupV5 } from "../../apis/consumer/leave-group.js";
-import { api as listOffsetsV9 } from "../../apis/consumer/list-offsets.js";
-import { api as offsetCommitV9 } from "../../apis/consumer/offset-commit.js";
-import { api as offsetFetchV9 } from "../../apis/consumer/offset-fetch.js";
-import { api as syncGroupV5 } from "../../apis/consumer/sync-group.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
-import { api as findCoordinatorV6 } from "../../apis/metadata/find-coordinator.js";
+import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
 import { UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Writer } from "../../protocol/writer.js";
-import { Base, kBootstrapBrokers, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
+import { Base, kAfterCreate, kBootstrapBrokers, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { defaultBaseOptions } from "../base/options.js";
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../callbacks.js";
 import { ensureMetric } from "../metrics.js";
@@ -72,6 +64,7 @@ export class Consumer extends Base {
             this.#metricActiveStreams = ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers_streams', 'Number of active Kafka consumers streams');
             this.topics.setMetric(ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers_topics', 'Number of topics being consumed'));
         }
+        this[kAfterCreate]('consumer');
     }
     get streamsCount() {
         return this.#streams.size;
@@ -153,7 +146,7 @@ export class Consumer extends Base {
             callback(validationError, undefined);
             return callback[kCallbackPromise];
         }
-        this.#fetch(options, callback);
+        consumerFetchesChannel.traceCallback(this.#fetch, 1, createDiagnosticContext({ client: this, operation: 'fetch', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
     commit(options, callback) {
@@ -168,7 +161,7 @@ export class Consumer extends Base {
             callback(validationError);
             return callback[kCallbackPromise];
         }
-        this.#commit(options, callback);
+        consumerCommitsChannel.traceCallback(this.#commit, 1, createDiagnosticContext({ client: this, operation: 'commit', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
     listOffsets(options, callback) {
@@ -183,7 +176,7 @@ export class Consumer extends Base {
             callback(validationError, undefined);
             return callback[kCallbackPromise];
         }
-        this.#listOffsets(options, callback);
+        consumerOffsetsChannel.traceCallback(this.#listOffsets, 1, createDiagnosticContext({ client: this, operation: 'listOffsets', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
     listCommittedOffsets(options, callback) {
@@ -198,7 +191,7 @@ export class Consumer extends Base {
             callback(validationError, undefined);
             return callback[kCallbackPromise];
         }
-        this.#listCommittedOffsets(options, callback);
+        consumerOffsetsChannel.traceCallback(this.#listCommittedOffsets, 1, createDiagnosticContext({ client: this, operation: 'listCommittedOffsets', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
     findGroupCoordinator(callback) {
@@ -258,37 +251,8 @@ export class Consumer extends Base {
         });
         return callback[kCallbackPromise];
     }
-    #consume(options, callback, trackTopics = true) {
-        // Subscribe all topics
-        let joinNeeded = this.memberId === null;
-        if (trackTopics) {
-            for (const topic of options.topics) {
-                if (this.topics.track(topic)) {
-                    joinNeeded = true;
-                }
-            }
-        }
-        // If we need to (re)join the group, do that first and then try again
-        if (joinNeeded) {
-            this.joinGroup(options, error => {
-                if (error) {
-                    callback(error, undefined);
-                    return;
-                }
-                this.#consume(options, callback, false);
-            });
-            return callback[kCallbackPromise];
-        }
-        // Create the stream and start consuming
-        const stream = new MessagesStream(this, options);
-        this.#streams.add(stream);
-        this.#metricActiveStreams?.inc();
-        stream.once('close', () => {
-            this.#metricActiveStreams?.dec();
-            this.#streams.delete(stream);
-        });
-        callback(null, stream);
-        return callback[kCallbackPromise];
+    #consume(options, callback) {
+        consumerConsumesChannel.traceCallback(this.#performConsume, 2, createDiagnosticContext({ client: this, operation: 'consume', options }), this, options, true, callback);
     }
     #fetch(options, callback) {
         this[kPerformWithRetry]('fetch', retryCallback => {
@@ -307,7 +271,13 @@ export class Consumer extends Base {
                         retryCallback(error, undefined);
                         return;
                     }
-                    fetchV17(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, FetchIsolationLevels[options.isolationLevel ?? this[kOptions].isolationLevel], 0, 0, options.topics, [], '', retryCallback);
+                    this[kGetApi]('Fetch', (error, api) => {
+                        if (error) {
+                            retryCallback(error, undefined);
+                            return;
+                        }
+                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, FetchIsolationLevels[options.isolationLevel ?? this[kOptions].isolationLevel], 0, 0, options.topics, [], '', retryCallback);
+                    });
                 });
             });
         }, callback, 0);
@@ -328,7 +298,13 @@ export class Consumer extends Base {
                     committedMetadata: null
                 });
             }
-            offsetCommitV9(connection, this.groupId, this.generationId, this.memberId, null, Array.from(topics.values()), groupCallback);
+            this[kGetApi]('OffsetCommit', (error, api) => {
+                if (error) {
+                    groupCallback(error, undefined);
+                    return;
+                }
+                api(connection, this.groupId, this.generationId, this.memberId, null, Array.from(topics.values()), groupCallback);
+            });
         }, error => {
             callback(error);
         });
@@ -369,7 +345,13 @@ export class Consumer extends Base {
                             retryCallback(error, undefined);
                             return;
                         }
-                        listOffsetsV9(connection, -1, FetchIsolationLevels[options.isolationLevel ?? this[kOptions].isolationLevel], Array.from(requests.values()), retryCallback);
+                        this[kGetApi]('ListOffsets', (error, api) => {
+                            if (error) {
+                                retryCallback(error, undefined);
+                                return;
+                            }
+                            api(connection, -1, FetchIsolationLevels[options.isolationLevel ?? this[kOptions].isolationLevel], Array.from(requests.values()), retryCallback);
+                        });
                     });
                 }, concurrentCallback, 0);
             }, (error, responses) => {
@@ -400,9 +382,15 @@ export class Consumer extends Base {
             topics.push({ name, partitionIndexes: partitions });
         }
         this.#performGroupOperation('listCommits', (connection, groupCallback) => {
-            offsetFetchV9(connection, 
-            // Note: once we start implementing KIP-848, the memberEpoch must be obtained
-            [{ groupId: this.groupId, memberId: this.memberId, memberEpoch: -1, topics }], false, groupCallback);
+            this[kGetApi]('OffsetFetch', (error, api) => {
+                if (error) {
+                    groupCallback(error, undefined);
+                    return;
+                }
+                api(connection, 
+                // Note: once we start implementing KIP-848, the memberEpoch must be obtained
+                [{ groupId: this.groupId, memberId: this.memberId, memberEpoch: -1, topics }], false, groupCallback);
+            });
         }, (error, response) => {
             if (error) {
                 callback(error, undefined);
@@ -427,6 +415,97 @@ export class Consumer extends Base {
             callback(null, this.#coordinatorId);
             return;
         }
+        consumerGroupChannel.traceCallback(this.#performFindGroupCoordinator, 0, createDiagnosticContext({ client: this, operation: 'findGroupCoordinator' }), this, callback);
+    }
+    #joinGroup(options, callback) {
+        consumerGroupChannel.traceCallback(this.#performJoinGroup, 1, createDiagnosticContext({ client: this, operation: 'joinGroup', options }), this, options, callback);
+    }
+    #leaveGroup(force, callback) {
+        consumerGroupChannel.traceCallback(this.#performLeaveGroup, 1, createDiagnosticContext({ client: this, operation: 'leaveGroup', force }), this, force, callback);
+    }
+    #syncGroup(callback) {
+        consumerGroupChannel.traceCallback(this.#performSyncGroup, 1, createDiagnosticContext({ client: this, operation: 'syncGroup' }), this, null, callback);
+    }
+    #heartbeat(options) {
+        const eventPayload = { groupId: this.groupId, memberId: this.memberId, generationId: this.generationId };
+        consumerHeartbeatChannel.traceCallback((this.#performDeduplicateGroupOperaton), 2, createDiagnosticContext({ client: this, operation: 'heartbeat' }), this, 'heartbeat', (connection, groupCallback) => {
+            // We have left the group in the meanwhile, abort
+            if (!this.#membershipActive) {
+                this.emitWithDebug('consumer:heartbeat', 'cancel', eventPayload);
+                return;
+            }
+            this.emitWithDebug('consumer:heartbeat', 'start', eventPayload);
+            this[kGetApi]('Heartbeat', (error, api) => {
+                if (error) {
+                    groupCallback(error, undefined);
+                    return;
+                }
+                api(connection, this.groupId, this.generationId, this.memberId, null, groupCallback);
+            });
+        }, error => {
+            // The heartbeat has been aborted elsewhere, ignore the response
+            if (this.#heartbeatInterval === null || !this.#membershipActive) {
+                this.emitWithDebug('consumer:heartbeat', 'cancel', eventPayload);
+                return;
+            }
+            if (error) {
+                this.#cancelHeartbeat();
+                if (this.#getRejoinError(error)) {
+                    this[kPerformWithRetry]('rejoinGroup', retryCallback => {
+                        this.#joinGroup(options, retryCallback);
+                    }, error => {
+                        if (error) {
+                            this.emitWithDebug(null, 'error', error);
+                        }
+                        this.emitWithDebug('consumer', 'rejoin');
+                    }, 0);
+                    return;
+                }
+                this.emitWithDebug('consumer:heartbeat', 'error', { ...eventPayload, error });
+                // Note that here we purposely do not return, since it was not a group related problem we schedule another heartbeat
+            }
+            else {
+                this.emitWithDebug('consumer:heartbeat', 'end', eventPayload);
+            }
+            this.#heartbeatInterval?.refresh();
+        });
+    }
+    #cancelHeartbeat() {
+        clearTimeout(this.#heartbeatInterval);
+        this.#heartbeatInterval = null;
+    }
+    #performConsume(options, trackTopics, callback) {
+        // Subscribe all topics
+        let joinNeeded = this.memberId === null;
+        if (trackTopics) {
+            for (const topic of options.topics) {
+                if (this.topics.track(topic)) {
+                    joinNeeded = true;
+                }
+            }
+        }
+        // If we need to (re)join the group, do that first and then try again
+        if (joinNeeded) {
+            this.joinGroup(options, error => {
+                if (error) {
+                    callback(error, undefined);
+                    return;
+                }
+                this.#performConsume(options, false, callback);
+            });
+            return;
+        }
+        // Create the stream and start consuming
+        const stream = new MessagesStream(this, options);
+        this.#streams.add(stream);
+        this.#metricActiveStreams?.inc();
+        stream.once('close', () => {
+            this.#metricActiveStreams?.dec();
+            this.#streams.delete(stream);
+        });
+        callback(null, stream);
+    }
+    #performFindGroupCoordinator(callback) {
         this[kPerformDeduplicated]('findGroupCoordinator', deduplicateCallback => {
             this[kPerformWithRetry]('findGroupCoordinator', retryCallback => {
                 this[kConnections].getFirstAvailable(this[kBootstrapBrokers], (error, connection) => {
@@ -434,7 +513,13 @@ export class Consumer extends Base {
                         retryCallback(error, undefined);
                         return;
                     }
-                    findCoordinatorV6(connection, FindCoordinatorKeyTypes.GROUP, [this.groupId], retryCallback);
+                    this[kGetApi]('FindCoordinator', (error, api) => {
+                        if (error) {
+                            retryCallback(error, undefined);
+                            return;
+                        }
+                        api(connection, FindCoordinatorKeyTypes.GROUP, [this.groupId], retryCallback);
+                    });
                 });
             }, (error, response) => {
                 if (error) {
@@ -447,7 +532,7 @@ export class Consumer extends Base {
             }, 0);
         }, callback);
     }
-    #joinGroup(options, callback) {
+    #performJoinGroup(options, callback) {
         if (!this.#membershipActive) {
             callback(null, undefined);
             return;
@@ -461,7 +546,13 @@ export class Consumer extends Base {
             });
         }
         this.#performDeduplicateGroupOperaton('joinGroup', (connection, groupCallback) => {
-            joinGroupV9(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', null, 'consumer', protocols, '', groupCallback);
+            this[kGetApi]('JoinGroup', (error, api) => {
+                if (error) {
+                    groupCallback(error, undefined);
+                    return;
+                }
+                api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', null, 'consumer', protocols, '', groupCallback);
+            });
         }, (error, response) => {
             if (!this.#membershipActive) {
                 callback(null, undefined);
@@ -469,7 +560,7 @@ export class Consumer extends Base {
             }
             if (error) {
                 if (this.#getRejoinError(error)) {
-                    this.#joinGroup(options, callback);
+                    this.#performJoinGroup(options, callback);
                     return;
                 }
                 callback(error, undefined);
@@ -483,14 +574,14 @@ export class Consumer extends Base {
                 this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
             }
             // Send a syncGroup request
-            this.#syncGroup(null, (error, response) => {
+            this.#syncGroup((error, response) => {
                 if (!this.#membershipActive) {
                     callback(null, undefined);
                     return;
                 }
                 if (error) {
                     if (this.#getRejoinError(error)) {
-                        this.#joinGroup(options, callback);
+                        this.#performJoinGroup(options, callback);
                         return;
                     }
                     callback(error, undefined);
@@ -512,7 +603,7 @@ export class Consumer extends Base {
             });
         });
     }
-    #leaveGroup(force, callback) {
+    #performLeaveGroup(force, callback) {
         if (!this.memberId) {
             callback(null);
             return;
@@ -536,13 +627,19 @@ export class Consumer extends Base {
                     return;
                 }
                 // All streams are closed, try the operation again without force
-                this.#leaveGroup(false, callback);
+                this.#performLeaveGroup(false, callback);
             });
             return;
         }
         this.#cancelHeartbeat();
         this.#performDeduplicateGroupOperaton('leaveGroup', (connection, groupCallback) => {
-            leaveGroupV5(connection, this.groupId, [{ memberId: this.memberId }], groupCallback);
+            this[kGetApi]('LeaveGroup', (error, api) => {
+                if (error) {
+                    groupCallback(error, undefined);
+                    return;
+                }
+                api(connection, this.groupId, [{ memberId: this.memberId }], groupCallback);
+            });
         }, error => {
             if (error) {
                 const unknownMemberError = error.findBy?.('unknownMemberId', true);
@@ -563,7 +660,7 @@ export class Consumer extends Base {
             callback(null);
         });
     }
-    #syncGroup(assignments, callback) {
+    #performSyncGroup(assignments, callback) {
         if (!this.#membershipActive) {
             callback(null, []);
             return;
@@ -587,7 +684,7 @@ export class Consumer extends Base {
                         callback(error, undefined);
                         return;
                     }
-                    this.#syncGroup(this.#roundRobinAssignments(metadata), callback);
+                    this.#performSyncGroup(this.#roundRobinAssignments(metadata), callback);
                 });
                 return;
             }
@@ -597,7 +694,13 @@ export class Consumer extends Base {
             }
         }
         this.#performDeduplicateGroupOperaton('syncGroup', (connection, groupCallback) => {
-            syncGroupV5(connection, this.groupId, this.generationId, this.memberId, null, 'consumer', this.#protocol, assignments, groupCallback);
+            this[kGetApi]('SyncGroup', (error, api) => {
+                if (error) {
+                    groupCallback(error, undefined);
+                    return;
+                }
+                api(connection, this.groupId, this.generationId, this.memberId, null, 'consumer', this.#protocol, assignments, groupCallback);
+            });
         }, (error, response) => {
             if (!this.#membershipActive) {
                 callback(null, undefined);
@@ -618,48 +721,6 @@ export class Consumer extends Base {
             callback(error, assignments);
         });
     }
-    #heartbeat(options) {
-        const eventPayload = { groupId: this.groupId, memberId: this.memberId, generationId: this.generationId };
-        this.#performDeduplicateGroupOperaton('heartbeat', (connection, groupCallback) => {
-            // We have left the group in the meanwhile, abort
-            if (!this.#membershipActive) {
-                this.emitWithDebug('consumer:heartbeat', 'cancel', eventPayload);
-                return;
-            }
-            this.emitWithDebug('consumer:heartbeat', 'start', eventPayload);
-            heartbeatV4(connection, this.groupId, this.generationId, this.memberId, null, groupCallback);
-        }, error => {
-            // The heartbeat has been aborted elsewhere, ignore the response
-            if (this.#heartbeatInterval === null || !this.#membershipActive) {
-                this.emitWithDebug('consumer:heartbeat', 'cancel', eventPayload);
-                return;
-            }
-            if (error) {
-                this.#cancelHeartbeat();
-                if (this.#getRejoinError(error)) {
-                    this[kPerformWithRetry]('rejoinGroup', retryCallback => {
-                        this.#joinGroup(options, retryCallback);
-                    }, error => {
-                        if (error) {
-                            this.emitWithDebug(null, 'error', error);
-                        }
-                        this.emitWithDebug('consumer', 'rejoin');
-                    }, 0);
-                    return;
-                }
-                this.emitWithDebug('consumer:heartbeat', 'error', { ...eventPayload, error });
-                // Note that here we purposely do not return, since it was not a group related problem we schedule another heartbeat
-            }
-            else {
-                this.emitWithDebug('consumer:heartbeat', 'end', eventPayload);
-            }
-            this.#heartbeatInterval?.refresh();
-        });
-    }
-    #cancelHeartbeat() {
-        clearTimeout(this.#heartbeatInterval);
-        this.#heartbeatInterval = null;
-    }
     #performDeduplicateGroupOperaton(operationId, operation, callback) {
         return this[kPerformDeduplicated](operationId, deduplicateCallback => {
             this.#performGroupOperation(operationId, operation, deduplicateCallback);

package/dist/clients/consumer/messages-stream.d.ts

@@ -5,6 +5,7 @@ import { type CallbackWithPromise } from '../callbacks.ts';
 import { type Consumer } from './consumer.ts';
 import { type CommitOptionsPartition, type ConsumeOptions } from './types.ts';
 export declare function noopDeserializer(data?: Buffer): Buffer | undefined;
+export declare function defaultCorruptedMessageHandler(): boolean;
 export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends Readable {
     #private;
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);