@platformatic/kafka 1.19.0 → 1.21.0

package/dist/clients/base/base.js

@@ -23,7 +23,6 @@ export const kClosed = Symbol('plt.kafka.base.closed');
 export const kListApis = Symbol('plt.kafka.base.listApis');
 export const kMetadata = Symbol('plt.kafka.base.metadata');
 export const kCheckNotClosed = Symbol('plt.kafka.base.checkNotClosed');
-export const kClearMetadata = Symbol('plt.kafka.base.clearMetadata');
 export const kPerformWithRetry = Symbol('plt.kafka.base.performWithRetry');
 export const kPerformDeduplicated = Symbol('plt.kafka.base.performDeduplicated');
 export const kValidateOptions = Symbol('plt.kafka.base.validateOptions');
@@ -118,7 +117,7 @@ export class Base extends EventEmitter {
             callback(validationError, undefined);
             return callback[kCallbackPromise];
         }
-        baseMetadataChannel.traceCallback(this[kMetadata], 1, createDiagnosticContext({ client: this, operation: 'metadata' }), this, options, callback);
+        this[kMetadata](options, callback);
         return callback[kCallbackPromise];
     }
     connectToBrokers(nodeIds, callback) {
@@ -209,101 +208,7 @@ export class Base extends EventEmitter {
         }, callback);
     }
     [kMetadata](options, callback) {
-        const expiralDate = Date.now() - (options.metadataMaxAge ?? this[kOptions].metadataMaxAge);
-        let topicsToFetch = [];
-        // Determine which topics we need to fetch
-        if (!this.#metadata || options.forceUpdate) {
-            topicsToFetch = options.topics;
-        }
-        else {
-            for (const topic of options.topics) {
-                const existingTopic = this.#metadata.topics.get(topic);
-                if (!existingTopic || existingTopic.lastUpdate < expiralDate) {
-                    topicsToFetch.push(topic);
-                }
-            }
-        }
-        // All topics are already up-to-date, simply return them
-        if (this.#metadata && !topicsToFetch.length && !options.forceUpdate) {
-            callback(null, {
-                ...this.#metadata,
-                topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
-            });
-            return;
-        }
-        const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
-        this[kPerformDeduplicated](
-        // Unique key to avoid mixing callbacks
-        `metadata-${options.topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
-            this[kPerformWithRetry]('metadata', retryCallback => {
-                this[kGetBootstrapConnection]((error, connection) => {
-                    if (error) {
-                        retryCallback(error, undefined);
-                        return;
-                    }
-                    this[kGetApi]('Metadata', (error, api) => {
-                        if (error) {
-                            retryCallback(error, undefined);
-                            return;
-                        }
-                        api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
-                    });
-                });
-            }, (error, metadata) => {
-                if (error) {
-                    const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
-                    // Stale metadata, we need to fetch everything again
-                    if (hasStaleMetadata) {
-                        this[kClearMetadata]();
-                        topicsToFetch = options.topics;
-                    }
-                    deduplicateCallback(error, undefined);
-                    return;
-                }
-                const lastUpdate = Date.now();
-                if (!this.#metadata) {
-                    this.#metadata = {
-                        id: metadata.clusterId,
-                        brokers: new Map(),
-                        topics: new Map(),
-                        lastUpdate
-                    };
-                }
-                else {
-                    this.#metadata.lastUpdate = lastUpdate;
-                }
-                const brokers = new Map();
-                // This should never change, but we act defensively here
-                for (const broker of metadata.brokers) {
-                    const { host, port } = broker;
-                    brokers.set(broker.nodeId, { host, port });
-                }
-                this.#metadata.brokers = brokers;
-                // Update all the topics in the cache
-                for (const { name, topicId: id, partitions: rawPartitions, isInternal } of metadata.topics) {
-                    /* c8 ignore next 3 - Sometimes internal topics might be returned by Kafka */
-                    if (isInternal) {
-                        continue;
-                    }
-                    const partitions = [];
-                    for (const rawPartition of rawPartitions.sort((a, b) => a.partitionIndex - b.partitionIndex)) {
-                        partitions[rawPartition.partitionIndex] = {
-                            leader: rawPartition.leaderId,
-                            leaderEpoch: rawPartition.leaderEpoch,
-                            replicas: rawPartition.replicaNodes
-                        };
-                    }
-                    this.#metadata.topics.set(name, { id, partitions, partitionsCount: rawPartitions.length, lastUpdate });
-                }
-                // Now build the object to return
-                const updatedMetadata = {
-                    ...this.#metadata,
-                    topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
-                };
-                this.emitWithDebug('client', 'metadata', updatedMetadata);
-                deduplicateCallback(null, updatedMetadata);
-            }, 0);
-        }, callback);
+        baseMetadataChannel.traceCallback(this.#performMetadata, 1, createDiagnosticContext({ client: this, operation: 'metadata' }), this, options, callback);
     }
     [kCheckNotClosed](callback) {
         if (this[kClosed]) {
@@ -313,7 +218,7 @@ export class Base extends EventEmitter {
         }
         return false;
     }
-    [kClearMetadata]() {
+    clearMetadata() {
         this.#metadata = undefined;
     }
     [kPerformWithRetry](operationId, operation, callback, attempt = 0, errors = [], shouldSkipRetry) {
@@ -430,6 +335,103 @@ export class Base extends EventEmitter {
         this[kClientType] = type;
         notifyCreation(type, this);
     }
+    #performMetadata(options, callback) {
+        const expiralDate = Date.now() - (options.metadataMaxAge ?? this[kOptions].metadataMaxAge);
+        let topicsToFetch = [];
+        // Determine which topics we need to fetch
+        if (!this.#metadata || options.forceUpdate) {
+            topicsToFetch = options.topics;
+        }
+        else {
+            for (const topic of options.topics) {
+                const existingTopic = this.#metadata.topics.get(topic);
+                if (!existingTopic || existingTopic.lastUpdate < expiralDate) {
+                    topicsToFetch.push(topic);
+                }
+            }
+        }
+        // All topics are already up-to-date, simply return them
+        if (this.#metadata && !topicsToFetch.length && !options.forceUpdate) {
+            callback(null, {
+                ...this.#metadata,
+                topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
+            });
+            return;
+        }
+        const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
+        this[kPerformDeduplicated](
+        // Unique key to avoid mixing callbacks
+        `metadata-${options.topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
+            this[kPerformWithRetry]('metadata', retryCallback => {
+                this[kGetBootstrapConnection]((error, connection) => {
+                    if (error) {
+                        retryCallback(error, undefined);
+                        return;
+                    }
+                    this[kGetApi]('Metadata', (error, api) => {
+                        if (error) {
+                            retryCallback(error, undefined);
+                            return;
+                        }
+                        api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
+                    });
+                });
+            }, (error, metadata) => {
+                if (error) {
+                    const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
+                    // Stale metadata, we need to fetch everything again
+                    if (hasStaleMetadata) {
+                        this.clearMetadata();
+                        topicsToFetch = options.topics;
+                    }
+                    deduplicateCallback(error, undefined);
+                    return;
+                }
+                const lastUpdate = Date.now();
+                if (!this.#metadata) {
+                    this.#metadata = {
+                        id: metadata.clusterId,
+                        brokers: new Map(),
+                        topics: new Map(),
+                        lastUpdate
+                    };
+                }
+                else {
+                    this.#metadata.lastUpdate = lastUpdate;
+                }
+                const brokers = new Map();
+                // This should never change, but we act defensively here
+                for (const broker of metadata.brokers) {
+                    const { host, port } = broker;
+                    brokers.set(broker.nodeId, { host, port });
+                }
+                this.#metadata.brokers = brokers;
+                // Update all the topics in the cache
+                for (const { name, topicId: id, partitions: rawPartitions, isInternal } of metadata.topics) {
+                    /* c8 ignore next 3 - Sometimes internal topics might be returned by Kafka */
+                    if (isInternal) {
+                        continue;
+                    }
+                    const partitions = [];
+                    for (const rawPartition of rawPartitions.sort((a, b) => a.partitionIndex - b.partitionIndex)) {
+                        partitions[rawPartition.partitionIndex] = {
+                            leader: rawPartition.leaderId,
+                            leaderEpoch: rawPartition.leaderEpoch,
+                            replicas: rawPartition.replicaNodes
+                        };
+                    }
+                    this.#metadata.topics.set(name, { id, partitions, partitionsCount: rawPartitions.length, lastUpdate });
+                }
+                // Now build the object to return
+                const updatedMetadata = {
+                    ...this.#metadata,
+                    topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
+                };
+                this.emitWithDebug('client', 'metadata', updatedMetadata);
+                deduplicateCallback(null, updatedMetadata);
+            }, 0);
+        }, callback);
+    }
     #forwardEvents(source, events) {
         for (const event of events) {
             source.on(event, (...args) => {

package/dist/clients/base/index.d.ts

@@ -1,3 +1,3 @@
-export { Base, kCheckNotClosed, kClearMetadata, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from './base.ts';
+export { Base, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from './base.ts';
 export * from './options.ts';
 export * from './types.ts';

package/dist/clients/base/index.js

@@ -1,3 +1,3 @@
-export { Base, kCheckNotClosed, kClearMetadata, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "./base.js";
+export { Base, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "./base.js";
 export * from "./options.js";
 export * from "./types.js";

package/dist/clients/consumer/consumer.js

@@ -6,7 +6,7 @@ import { INT32_SIZE } from "../../protocol/definitions.js";
 import { Reader } from "../../protocol/reader.js";
 import { Writer } from "../../protocol/writer.js";
 import { kAutocommit, kRefreshOffsetsAndFetch } from "../../symbols.js";
-import { Base, kAfterCreate, kCheckNotClosed, kClearMetadata, kClosed, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
+import { Base, kAfterCreate, kCheckNotClosed, kClosed, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { defaultBaseOptions } from "../base/options.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
@@ -271,7 +271,7 @@ export class Consumer extends Base {
             // Now gather the last committed offsets from each stream
             const committeds = new Map();
             for (const stream of this.#streams) {
-                for (const [topic, offset] of stream.committedOffsets) {
+                for (const [topic, offset] of stream.offsetsCommitted) {
                     committeds.set(topic, offset);
                 }
             }
@@ -933,6 +933,8 @@ export class Consumer extends Base {
                 callback(error, undefined);
                 return;
             }
+            // This is for Azure Event Hubs compatibility, which does not respond with an error on the first join
+            this.memberId = response.memberId;
             this.generationId = response.generationId;
             this.#isLeader = response.leader === this.memberId;
             this.#protocol = response.protocolName;
@@ -1212,7 +1214,7 @@ export class Consumer extends Base {
     }
     #handleMetadataError(error) {
         if (error && error?.findBy('hasStaleMetadata', true)) {
-            this[kClearMetadata]();
+            this.clearMetadata();
         }
         return error;
     }

package/dist/clients/consumer/messages-stream.d.ts

@@ -11,6 +11,9 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     #private;
     [kInstance]: number;
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
+    get offsetsToFetch(): Map<string, bigint>;
+    get offsetsToCommit(): Map<string, CommitOptionsPartition>;
+    get offsetsCommitted(): Map<string, bigint>;
     get committedOffsets(): Map<string, bigint>;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
@@ -20,6 +23,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     pause(): this;
     addListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     addListener(event: 'fetch', listener: () => void): this;
+    addListener(event: 'offsets', listener: () => void): this;
     addListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     addListener(event: 'close', listener: () => void): this;
     addListener(event: 'end', listener: () => void): this;
@@ -29,6 +33,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     addListener(event: 'resume', listener: () => void): this;
     on(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     on(event: 'fetch', listener: () => void): this;
+    on(event: 'offsets', listener: () => void): this;
     on(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     on(event: 'close', listener: () => void): this;
     on(event: 'end', listener: () => void): this;
@@ -38,6 +43,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     on(event: 'resume', listener: () => void): this;
     once(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     once(event: 'fetch', listener: () => void): this;
+    once(event: 'offsets', listener: () => void): this;
     once(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     once(event: 'close', listener: () => void): this;
     once(event: 'end', listener: () => void): this;
@@ -47,6 +53,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     once(event: 'resume', listener: () => void): this;
     prependListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     prependListener(event: 'fetch', listener: () => void): this;
+    prependListener(event: 'offsets', listener: () => void): this;
     prependListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     prependListener(event: 'close', listener: () => void): this;
     prependListener(event: 'end', listener: () => void): this;
@@ -56,6 +63,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     prependListener(event: 'resume', listener: () => void): this;
     prependOnceListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     prependOnceListener(event: 'fetch', listener: () => void): this;
+    prependOnceListener(event: 'offsets', listener: () => void): this;
     prependOnceListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     prependOnceListener(event: 'close', listener: () => void): this;
     prependOnceListener(event: 'end', listener: () => void): this;

package/dist/clients/consumer/messages-stream.js

@@ -53,7 +53,7 @@ export class MessagesStream extends Readable {
         if (!offsets && mode === MessagesStreamModes.MANUAL) {
             throw new UserError('Must specify offsets when the stream mode is MANUAL.');
         }
-        /* c8 ignore next - Unless is initialized directly, highWaterMark is always defined */
+        /* c8 ignore next 4 - Unless is initialized directly, highWaterMark is always defined */
         super({
             objectMode: true,
             highWaterMark: maxFetches ?? options.highWaterMark ?? defaultConsumerOptions.highWaterMark
@@ -98,6 +98,7 @@ export class MessagesStream extends Readable {
         // will have changed so we may have gone from last  with no assignments to
         // having some.
         this.#consumer.on('consumer:group:join', () => {
+            this.#offsetsCommitted.clear();
             this.#refreshOffsets((error) => {
                 /* c8 ignore next 4 - Hard to test */
                 if (error) {
@@ -112,6 +113,20 @@ export class MessagesStream extends Readable {
         }
         notifyCreation('messages-stream', this);
     }
+    /* c8 ignore next 3 - Simple getter */
+    get offsetsToFetch() {
+        return this.#offsetsToFetch;
+    }
+    /* c8 ignore next 3 - Simple getter */
+    get offsetsToCommit() {
+        return this.#offsetsToCommit;
+    }
+    /* c8 ignore next 3 - Simple getter */
+    get offsetsCommitted() {
+        return this.#offsetsCommitted;
+    }
+    // TODO: This is deprecated alias, remove in future major version
+    /* c8 ignore next 3 - Simple getter */
     get committedOffsets() {
         return this.#offsetsCommitted;
     }
@@ -525,6 +540,20 @@ export class MessagesStream extends Readable {
                 }
             }
         }
+        // Rebuild the list of offsetsCommitted (which is used for consumer lag) out of the offsets to fetch
+        for (const topic of this.#topics) {
+            const assignment = this.#assignmentsForTopic(topic);
+            // This consumer has no assignment for the topic, continue
+            if (!assignment) {
+                continue;
+            }
+            const partitions = assignment.partitions;
+            for (const partition of partitions) {
+                const committed = this.#offsetsToFetch.get(`${topic}:${partition}`);
+                this.#offsetsCommitted.set(`${topic}:${partition}`, committed - 1n);
+            }
+        }
+        this.emit('offsets');
         callback(null);
     }
     #assignmentsForTopic(topic) {

package/dist/clients/producer/producer.js

@@ -4,7 +4,7 @@ import { createDiagnosticContext, producerInitIdempotentChannel, producerSendsCh
 import { UserError } from "../../errors.js";
 import { murmur2 } from "../../protocol/murmur2.js";
 import { NumericMap } from "../../utils.js";
-import { Base, kAfterCreate, kCheckNotClosed, kClearMetadata, kClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
+import { Base, kAfterCreate, kCheckNotClosed, kClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { produceOptionsValidator, producerOptionsValidator, sendOptionsValidator } from "./options.js";
 // Don't move this function as being in the same file will enable V8 to remove.
@@ -311,7 +311,7 @@ export class Producer extends Base {
                     // since the partition is already set, it should attempt on the new destination
                     const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
                     if (hasStaleMetadata && repeatOnStaleMetadata) {
-                        this[kClearMetadata]();
+                        this.clearMetadata();
                         this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, callback);
                         return;
                     }

package/dist/network/connection.js

@@ -61,7 +61,7 @@ export class Connection extends EventEmitter {
         this.#correlationId = 0;
         this.#nextMessage = 0;
         this.#afterDrainRequests = [];
-        this.#requestsQueue = fastq((op, cb) => op(cb), this.#options.maxInflights);
+        this.#requestsQueue = fastq((op, cb) => op(cb), this.#options.maxInflights ?? defaultOptions.maxInflights);
         this.#inflightRequests = new Map();
         this.#responseBuffer = new DynamicBuffer();
         this.#responseReader = new Reader(this.#responseBuffer);
@@ -230,7 +230,7 @@ export class Connection extends EventEmitter {
         catch (err) {
             diagnostic.error = err;
             connectionsApiChannel.error.publish(diagnostic);
-            throw err;
+            return callback(err, undefined);
         }
         writer.appendFrom(payload).prependLength();
         const request = {

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.19.0";
+export const version = "1.21.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.19.0",
+  "version": "1.21.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",