@platformatic/kafka 1.23.0 → 1.24.0

package/dist/clients/base/base.js

@@ -54,11 +54,11 @@ export class Base extends EventEmitter {
         this[kApis] = [];
         // Validate options
         this[kOptions] = Object.assign({}, defaultBaseOptions, options);
+        this[kValidateOptions](this[kOptions], baseOptionsValidator, '/options');
+        this[kClientId] = options.clientId;
         if (typeof this[kOptions].retries === 'boolean') {
             this[kOptions].retries = this[kOptions].retries ? Number.POSITIVE_INFINITY : 0;
         }
-        this[kValidateOptions](this[kOptions], baseOptionsValidator, '/options');
-        this[kClientId] = options.clientId;
         // Initialize bootstrap brokers
         this[kBootstrapBrokers] = [];
         for (const broker of options.bootstrapBrokers) {
@@ -114,7 +114,7 @@ export class Base extends EventEmitter {
         }
         const validationError = this[kValidateOptions](options, metadataOptionsValidator, '/options', false);
         if (validationError) {
-            callback(validationError, undefined);
+            callback(validationError);
             return callback[kCallbackPromise];
         }
         this[kMetadata](options, callback);
@@ -127,7 +127,7 @@ export class Base extends EventEmitter {
         // Fetch the metadata
         this[kMetadata]({ topics: [] }, (error, metadata) => {
             if (error) {
-                callback(error, undefined);
+                callback(error);
                 return;
             }
             let nodes = [];
@@ -144,14 +144,14 @@ export class Base extends EventEmitter {
             runConcurrentCallbacks('Connecting to brokers failed.', nodes, (nodeId, concurrentCallback) => {
                 this[kGetConnection](metadata.brokers.get(nodeId), (error, connection) => {
                     if (error) {
-                        concurrentCallback(error, undefined);
+                        concurrentCallback(error);
                         return;
                     }
                     concurrentCallback(null, [nodeId, connection]);
                 });
             }, (error, connections) => {
                 if (error) {
-                    callback(error, undefined);
+                    callback(error);
                     return;
                 }
                 return callback(null, new Map(connections));
@@ -192,7 +192,7 @@ export class Base extends EventEmitter {
             this[kPerformWithRetry]('listApis', retryCallback => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
-                        retryCallback(error, undefined);
+                        retryCallback(error);
                         return;
                     }
                     // We use V3 to be able to get APIS from Kafka 2.4.0+
@@ -200,7 +200,7 @@ export class Base extends EventEmitter {
                 });
             }, (error, metadata) => {
                 if (error) {
-                    deduplicateCallback(error, undefined);
+                    deduplicateCallback(error);
                     return;
                 }
                 deduplicateCallback(null, metadata.apiKeys);
@@ -235,7 +235,7 @@ export class Base extends EventEmitter {
                     function onClose() {
                         clearTimeout(timeout);
                         errors.push(new UserError(`Client closed while retrying ${operationId}.`));
-                        callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors), undefined);
+                        callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors));
                     }
                     const timeout = setTimeout(() => {
                         this.removeListener('client:close', onClose);
@@ -245,10 +245,10 @@ export class Base extends EventEmitter {
                 }
                 else {
                     if (attempt === 0) {
-                        callback(error, undefined);
+                        callback(error);
                         return;
                     }
-                    callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors), undefined);
+                    callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors));
                 }
                 return;
             }
@@ -280,7 +280,7 @@ export class Base extends EventEmitter {
         if (!this[kApis].length) {
             this[kListApis]((error, apis) => {
                 if (error) {
-                    callback(error, undefined);
+                    callback(error);
                     return;
                 }
                 this[kApis] = apis;
@@ -290,7 +290,7 @@ export class Base extends EventEmitter {
         }
         const api = this[kApis].find(api => api.name === name);
         if (!api) {
-            callback(new UnsupportedApiError(`Unsupported API ${name}.`), undefined);
+            callback(new UnsupportedApiError(`Unsupported API ${name}.`));
             return;
         }
         const { minVersion, maxVersion } = api;
@@ -303,7 +303,7 @@ export class Base extends EventEmitter {
                 return;
             }
         }
-        callback(new UnsupportedApiError(`No usable implementation found for API ${name}.`, { minVersion, maxVersion }), undefined);
+        callback(new UnsupportedApiError(`No usable implementation found for API ${name}.`, { minVersion, maxVersion }));
     }
     [kGetConnection](broker, callback) {
         this[kConnections].get(broker, callback);
@@ -339,12 +339,13 @@ export class Base extends EventEmitter {
     #performMetadata(options, callback) {
         const expiralDate = Date.now() - (options.metadataMaxAge ?? this[kOptions].metadataMaxAge);
         let topicsToFetch = [];
+        const topics = options.topics ?? [];
         // Determine which topics we need to fetch
         if (!this.#metadata || options.forceUpdate) {
-            topicsToFetch = options.topics;
+            topicsToFetch = topics;
         }
         else {
-            for (const topic of options.topics) {
+            for (const topic of topics) {
                 const existingTopic = this.#metadata.topics.get(topic);
                 if (!existingTopic || existingTopic.lastUpdate < expiralDate) {
                     topicsToFetch.push(topic);
@@ -355,23 +356,23 @@ export class Base extends EventEmitter {
         if (this.#metadata && !topicsToFetch.length && !options.forceUpdate) {
             callback(null, {
                 ...this.#metadata,
-                topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
+                topics: new Map(topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
             });
             return;
         }
         const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
         this[kPerformDeduplicated](
         // Unique key to avoid mixing callbacks
-        `metadata-${options.topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
+        `metadata-${topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
             this[kPerformWithRetry]('metadata', retryCallback => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
-                        retryCallback(error, undefined);
+                        retryCallback(error);
                         return;
                     }
                     this[kGetApi]('Metadata', (error, api) => {
                         if (error) {
-                            retryCallback(error, undefined);
+                            retryCallback(error);
                             return;
                         }
                         api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
@@ -383,9 +384,9 @@ export class Base extends EventEmitter {
                     // Stale metadata, we need to fetch everything again
                     if (hasStaleMetadata) {
                         this.clearMetadata();
-                        topicsToFetch = options.topics;
+                        topicsToFetch = topics;
                     }
-                    deduplicateCallback(error, undefined);
+                    deduplicateCallback(error);
                     return;
                 }
                 const lastUpdate = Date.now();
@@ -404,8 +405,8 @@ export class Base extends EventEmitter {
                 const brokers = new Map();
                 // This should never change, but we act defensively here
                 for (const broker of metadata.brokers) {
-                    const { host, port } = broker;
-                    brokers.set(broker.nodeId, { host, port });
+                    const { host, port, rack } = broker;
+                    brokers.set(broker.nodeId, { host, port, rack });
                 }
                 this.#metadata.brokers = brokers;
                 // Update all the topics in the cache
@@ -419,7 +420,9 @@ export class Base extends EventEmitter {
                         partitions[rawPartition.partitionIndex] = {
                             leader: rawPartition.leaderId,
                             leaderEpoch: rawPartition.leaderEpoch,
-                            replicas: rawPartition.replicaNodes
+                            replicas: rawPartition.replicaNodes,
+                            isr: rawPartition.isrNodes,
+                            offlineReplicas: rawPartition.offlineReplicas
                         };
                     }
                     this.#metadata.topics.set(name, { id, partitions, partitionsCount: rawPartitions.length, lastUpdate });
@@ -427,7 +430,7 @@ export class Base extends EventEmitter {
                 // Now build the object to return
                 const updatedMetadata = {
                     ...this.#metadata,
-                    topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
+                    topics: new Map(topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
                 };
                 this.emitWithDebug('client', 'metadata', updatedMetadata);
                 deduplicateCallback(null, updatedMetadata);

package/dist/clients/base/options.d.ts

@@ -92,7 +92,7 @@ export declare const baseOptionsSchema: {
             properties: {
                 mechanism: {
                     type: string;
-                    enum: import("../../apis/enumerations.ts").SASLMechanismValue[];
+                    enum: ("PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER" | "GSSAPI")[];
                 };
                 username: {
                     oneOf: ({
@@ -185,14 +185,11 @@ export declare const metadataOptionsSchema: {
             minimum: number;
         };
     };
-    required: string[];
     additionalProperties: boolean;
 };
 export declare const baseOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;
-export declare const metadataOptionsValidator: import("ajv").ValidateFunction<{
-    [x: string]: {};
-}>;
+export declare const metadataOptionsValidator: import("ajv").ValidateFunction<unknown>;
 export declare const defaultPort = 9092;
 export declare const defaultBaseOptions: Partial<BaseOptions>;

package/dist/clients/base/options.js

@@ -66,7 +66,6 @@ export const metadataOptionsSchema = {
         forceUpdate: { type: 'boolean' },
         metadataMaxAge: { type: 'number', minimum: 0 }
     },
-    required: ['topics'],
     additionalProperties: false
 };
 export const baseOptionsValidator = ajv.compile(baseOptionsSchema);

package/dist/clients/base/types.d.ts

@@ -1,4 +1,5 @@
 import { type Broker, type ConnectionOptions } from '../../network/connection.ts';
+import { type NullableString } from '../../protocol/definitions.ts';
 import { type Metrics } from '../metrics.ts';
 export interface TopicWithPartitionAndOffset {
     topic: string;
@@ -9,6 +10,8 @@ export interface ClusterPartitionMetadata {
     leader: number;
     leaderEpoch: number;
     replicas: number[];
+    isr: number[];
+    offlineReplicas: number[];
 }
 export interface ClusterTopicMetadata {
     id: string;
@@ -16,9 +19,12 @@ export interface ClusterTopicMetadata {
     partitionsCount: number;
     lastUpdate: number;
 }
+export interface BrokerWithRack extends Broker {
+    rack: NullableString;
+}
 export interface ClusterMetadata {
     id: string;
-    brokers: Map<number, Broker>;
+    brokers: Map<number, BrokerWithRack>;
     controllerId: number;
     topics: Map<string, ClusterTopicMetadata>;
     lastUpdate: number;
@@ -35,7 +41,7 @@ export interface BaseOptions extends ConnectionOptions {
     metrics?: Metrics;
 }
 export interface MetadataOptions {
-    topics: string[];
+    topics?: string[];
     autocreateTopics?: boolean;
     forceUpdate?: boolean;
     metadataMaxAge?: number;

package/dist/clients/consumer/consumer.d.ts

@@ -4,7 +4,7 @@ import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { Base, kFetchConnections } from '../base/base.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
-import { type CommitOptions, type ConsumeOptions, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
 export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ConsumerOptions<Key, Value, HeaderKey, HeaderValue>> {
     #private;
     groupId: string;
@@ -16,6 +16,62 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     constructor(options: ConsumerOptions<Key, Value, HeaderKey, HeaderValue>);
     get streamsCount(): number;
     get lastHeartbeat(): Date | null;
+    get coordinatorId(): number | null;
+    addListener(event: 'consumer:group:join', listener: (payload: ConsumerGroupJoinPayload) => void): this;
+    addListener(event: 'consumer:group:leave', listener: (payload: ConsumerGroupLeavePayload) => void): this;
+    addListener(event: 'consumer:group:rejoin', listener: () => void): this;
+    addListener(event: 'consumer:group:rebalance', listener: (payload: ConsumerGroupRebalancePayload) => void): this;
+    addListener(event: 'consumer:heartbeat:start', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    addListener(event: 'consumer:heartbeat:cancel', listener: (payload: ConsumerHeartbeatPayload) => void): this;
+    addListener(event: 'consumer:heartbeat:end', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    addListener(event: 'consumer:heartbeat:error', listener: (payload: ConsumerHeartbeatErrorPayload) => void): this;
+    addListener(event: 'consumer:lag', listener: (lag: Offsets) => void): this;
+    addListener(event: 'consumer:lag:error', listener: (error: Error) => void): this;
+    addListener(event: 'error', listener: (error: Error) => void): this;
+    on(event: 'consumer:group:join', listener: (payload: ConsumerGroupJoinPayload) => void): this;
+    on(event: 'consumer:group:leave', listener: (payload: ConsumerGroupLeavePayload) => void): this;
+    on(event: 'consumer:group:rejoin', listener: () => void): this;
+    on(event: 'consumer:group:rebalance', listener: (payload: ConsumerGroupRebalancePayload) => void): this;
+    on(event: 'consumer:heartbeat:start', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    on(event: 'consumer:heartbeat:cancel', listener: (payload: ConsumerHeartbeatPayload) => void): this;
+    on(event: 'consumer:heartbeat:end', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    on(event: 'consumer:heartbeat:error', listener: (payload: ConsumerHeartbeatErrorPayload) => void): this;
+    on(event: 'consumer:lag', listener: (lag: Offsets) => void): this;
+    on(event: 'consumer:lag:error', listener: (error: Error) => void): this;
+    on(event: 'error', listener: (error: Error) => void): this;
+    once(event: 'consumer:group:join', listener: (payload: ConsumerGroupJoinPayload) => void): this;
+    once(event: 'consumer:group:leave', listener: (payload: ConsumerGroupLeavePayload) => void): this;
+    once(event: 'consumer:group:rejoin', listener: () => void): this;
+    once(event: 'consumer:group:rebalance', listener: (payload: ConsumerGroupRebalancePayload) => void): this;
+    once(event: 'consumer:heartbeat:start', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    once(event: 'consumer:heartbeat:cancel', listener: (payload: ConsumerHeartbeatPayload) => void): this;
+    once(event: 'consumer:heartbeat:end', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    once(event: 'consumer:heartbeat:error', listener: (payload: ConsumerHeartbeatErrorPayload) => void): this;
+    once(event: 'consumer:lag', listener: (lag: Offsets) => void): this;
+    once(event: 'consumer:lag:error', listener: (error: Error) => void): this;
+    once(event: 'error', listener: (error: Error) => void): this;
+    prependListener(event: 'consumer:group:join', listener: (payload: ConsumerGroupJoinPayload) => void): this;
+    prependListener(event: 'consumer:group:leave', listener: (payload: ConsumerGroupLeavePayload) => void): this;
+    prependListener(event: 'consumer:group:rejoin', listener: () => void): this;
+    prependListener(event: 'consumer:group:rebalance', listener: (payload: ConsumerGroupRebalancePayload) => void): this;
+    prependListener(event: 'consumer:heartbeat:start', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    prependListener(event: 'consumer:heartbeat:cancel', listener: (payload: ConsumerHeartbeatPayload) => void): this;
+    prependListener(event: 'consumer:heartbeat:end', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    prependListener(event: 'consumer:heartbeat:error', listener: (payload: ConsumerHeartbeatErrorPayload) => void): this;
+    prependListener(event: 'consumer:lag', listener: (lag: Offsets) => void): this;
+    prependListener(event: 'consumer:lag:error', listener: (error: Error) => void): this;
+    prependListener(event: 'error', listener: (error: Error) => void): this;
+    prependOnceListener(event: 'consumer:group:join', listener: (payload: ConsumerGroupJoinPayload) => void): this;
+    prependOnceListener(event: 'consumer:group:leave', listener: (payload: ConsumerGroupLeavePayload) => void): this;
+    prependOnceListener(event: 'consumer:group:rejoin', listener: () => void): this;
+    prependOnceListener(event: 'consumer:group:rebalance', listener: (payload: ConsumerGroupRebalancePayload) => void): this;
+    prependOnceListener(event: 'consumer:heartbeat:start', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    prependOnceListener(event: 'consumer:heartbeat:cancel', listener: (payload: ConsumerHeartbeatPayload) => void): this;
+    prependOnceListener(event: 'consumer:heartbeat:end', listener: (payload?: ConsumerHeartbeatPayload) => void): this;
+    prependOnceListener(event: 'consumer:heartbeat:error', listener: (payload: ConsumerHeartbeatErrorPayload) => void): this;
+    prependOnceListener(event: 'consumer:lag', listener: (lag: Offsets) => void): this;
+    prependOnceListener(event: 'consumer:lag:error', listener: (error: Error) => void): this;
+    prependOnceListener(event: 'error', listener: (error: Error) => void): this;
     close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     close(force?: boolean): Promise<void>;
     isActive(): boolean;
@@ -38,7 +94,7 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     findGroupCoordinator(callback: CallbackWithPromise<number>): void;
     findGroupCoordinator(): Promise<number>;
     joinGroup(options: GroupOptions, callback: CallbackWithPromise<string>): void;
-    joinGroup(options: GroupOptions): Promise<string>;
+    joinGroup(options?: GroupOptions): Promise<string>;
     leaveGroup(force?: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     leaveGroup(force?: boolean): Promise<void>;
 }