@platformatic/kafka 0.1.1 → 0.3.0

package/README.md

@@ -24,13 +24,13 @@ npm install @platformatic/kafka
 ### Producer
 
 ```typescript
-import { Producer, stringSerializer } from '@platformatic/kafka'
+import { Producer, stringSerializers } from '@platformatic/kafka'
 
 // Create a producer with string serialisers
 const producer = new Producer({
   clientId: 'my-producer',
   bootstrapBrokers: ['localhost:9092'],
-  serializers: stringSerializer
+  serializers: stringSerializers
 })
 
 // Send messages
@@ -52,7 +52,7 @@ await producer.close()
 ### Consumer
 
 ```typescript
-import { Consumer, stringDeserializer } from '@platformatic/kafka'
+import { Consumer, stringDeserializers } from '@platformatic/kafka'
 import { forEach } from 'hwp'
 
 // Create a consumer with string deserialisers
@@ -60,7 +60,7 @@ const consumer = new Consumer({
   groupId: 'my-consumer-group',
   clientId: 'my-consumer',
   bootstrapBrokers: ['localhost:9092'],
-  deserializers: stringDeserializer
+  deserializers: stringDeserializers
 })
 
 // Create a consumer stream
@@ -83,10 +83,14 @@ for await (const message of stream) {
 }
 
 // Option 3: Concurrent processing
-await forEach(stream, async message => {
-  console.log(`Received: ${message.key} -> ${message.value}`)
-  // Process message...
-}, 16) // 16 is the concurrency level
+await forEach(
+  stream,
+  async message => {
+    console.log(`Received: ${message.key} -> ${message.value}`)
+    // Process message...
+  },
+  16
+) // 16 is the concurrency level
 
 // Close the consumer when done
 await consumer.close()
@@ -164,7 +168,7 @@ type Strings = string[]
 
 const producer = new Producer({
   clientId: 'my-producer',
-  bootstrapBrokers: ['localhost:29092'],
+  bootstrapBrokers: ['localhost:9092'],
   serializers: {
     key: stringSerializer,
     value: jsonSerializer<Strings>
@@ -174,7 +178,7 @@ const producer = new Producer({
 const consumer = new Consumer({
   groupId: 'my-consumer-group',
   clientId: 'my-consumer',
-  bootstrapBrokers: ['localhost:29092'],
+  bootstrapBrokers: ['localhost:9092'],
   deserializers: {
     key: stringDeserializer,
     value: jsonDeserializer<Strings>
@@ -259,6 +263,7 @@ Many of the methods accept the same options as the client's constructors. The co
 - [Consumer API](./docs/consumer.md)
 - [Admin API](./docs/admin.md)
 - [Base Client](./docs/base.md)
+- [Metrics](./docs/metrics.md)
 - [Other APIs and Types](./docs/other.md)
 
 ## Requirements

package/dist/apis/admin/consumer-group-describe.js

@@ -76,7 +76,7 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                                 return {
                                     topicId: r.readUUID(),
                                     topicName: r.readString(),
-                                    partitions: r.readArray(() => r.readInt32())
+                                    partitions: r.readArray(() => r.readInt32(), true, false)
                                 };
                             })
                         },
@@ -85,7 +85,7 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                                 return {
                                     topicId: r.readUUID(),
                                     topicName: r.readString(),
-                                    partitions: r.readArray(() => r.readInt32())
+                                    partitions: r.readArray(() => r.readInt32(), true, false)
                                 };
                             })
                         }

package/dist/apis/producer/txn-offset-commit.js

@@ -28,16 +28,13 @@ export function createRequest(transactionalId, groupId, producerId, producerEpoc
         .appendString(memberId, true)
         .appendString(groupInstanceId, true)
         .appendArray(topics, (w, t) => {
-        w.appendString(t.name, true)
-            .appendArray(t.partitions, (w, p) => {
+        w.appendString(t.name, true).appendArray(t.partitions, (w, p) => {
             w.appendInt32(p.partitionIndex)
                 .appendInt64(p.committedOffset)
                 .appendInt32(p.committedLeaderEpoch)
-                .appendString(p.committedMetadata, true)
-                .appendTaggedFields(); // Add tagged fields for partitions
-        }, true, true)
-            .appendTaggedFields(); // Add tagged fields for topics
-    }, true, true)
+                .appendString(p.committedMetadata, true);
+        });
+    })
         .appendTaggedFields();
 }
 /*

package/dist/clients/base/base.d.ts

@@ -4,11 +4,13 @@ import { type Callback } from '../../apis/definitions.ts';
 import { ConnectionPool } from '../../network/connection-pool.ts';
 import { type Broker } from '../../network/connection.ts';
 import { type CallbackWithPromise } from '../callbacks.ts';
+import { type Metrics } from '../metrics.ts';
 import { type BaseOptions, type ClusterMetadata, type MetadataOptions } from './types.ts';
 export declare const kClientId: unique symbol;
 export declare const kBootstrapBrokers: unique symbol;
 export declare const kOptions: unique symbol;
 export declare const kConnections: unique symbol;
+export declare const kFetchConnections: unique symbol;
 export declare const kCreateConnectionPool: unique symbol;
 export declare const kClosed: unique symbol;
 export declare const kMetadata: unique symbol;
@@ -19,7 +21,9 @@ export declare const kPerformWithRetry: unique symbol;
 export declare const kPerformDeduplicated: unique symbol;
 export declare const kValidateOptions: unique symbol;
 export declare const kInspect: unique symbol;
+export declare const kFormatValidationErrors: unique symbol;
 export declare const kInstance: unique symbol;
+export declare const kPrometheus: unique symbol;
 export declare class Base<OptionsType extends BaseOptions> extends EventEmitter {
     #private;
     [kInstance]: number;
@@ -28,6 +32,7 @@ export declare class Base<OptionsType extends BaseOptions> extends EventEmitter
     [kOptions]: OptionsType;
     [kConnections]: ConnectionPool;
     [kClosed]: boolean;
+    [kPrometheus]: Metrics | undefined;
     constructor(options: OptionsType);
     get clientId(): string;
     get closed(): boolean;
@@ -45,4 +50,5 @@ export declare class Base<OptionsType extends BaseOptions> extends EventEmitter
     [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
     [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
     [kInspect](...args: unknown[]): void;
+    [kFormatValidationErrors](validator: ValidateFunction<unknown>, targetName: string): string;
 }

package/dist/clients/base/base.js

@@ -8,9 +8,10 @@ import { baseOptionsValidator, defaultBaseOptions, defaultPort, metadataOptionsV
 export const kClientId = Symbol('plt.kafka.base.clientId');
 export const kBootstrapBrokers = Symbol('plt.kafka.base.bootstrapBrokers');
 export const kOptions = Symbol('plt.kafka.base.options');
-export const kConnections = Symbol('plt.kafka.base.kConnections');
-export const kCreateConnectionPool = Symbol('plt.kafka.base.kCreateConnectionPool');
-export const kClosed = Symbol('plt.kafka.base.kClosed');
+export const kConnections = Symbol('plt.kafka.base.connections');
+export const kFetchConnections = Symbol('plt.kafka.base.fetchCnnections');
+export const kCreateConnectionPool = Symbol('plt.kafka.base.createConnectionPool');
+export const kClosed = Symbol('plt.kafka.base.closed');
 export const kMetadata = Symbol('plt.kafka.base.metadata');
 export const kCheckNotClosed = Symbol('plt.kafka.base.checkNotClosed');
 export const kClearMetadata = Symbol('plt.kafka.base.clearMetadata');
@@ -19,7 +20,9 @@ export const kPerformWithRetry = Symbol('plt.kafka.base.performWithRetry');
 export const kPerformDeduplicated = Symbol('plt.kafka.base.performDeduplicated');
 export const kValidateOptions = Symbol('plt.kafka.base.validateOptions');
 export const kInspect = Symbol('plt.kafka.base.inspect');
+export const kFormatValidationErrors = Symbol('plt.kafka.base.formatValidationErrors');
 export const kInstance = Symbol('plt.kafka.base.instance');
+export const kPrometheus = Symbol('plt.kafka.base.prometheus');
 let currentInstance = 0;
 export class Base extends EventEmitter {
     // This is just used for debugging
@@ -30,6 +33,7 @@ export class Base extends EventEmitter {
     [kOptions];
     [kConnections];
     [kClosed];
+    [kPrometheus];
     #metadata;
     #inflightDeduplications;
     constructor(options) {
@@ -48,6 +52,10 @@ export class Base extends EventEmitter {
         this[kConnections] = this[kCreateConnectionPool]();
         this[kClosed] = false;
         this.#inflightDeduplications = new Map();
+        // Initialize metrics
+        if (options.metrics) {
+            this[kPrometheus] = options.metrics;
+        }
     }
     /* c8 ignore next 3 */
     get clientId() {
@@ -226,7 +234,7 @@ export class Base extends EventEmitter {
         }
         const valid = validator(target);
         if (!valid) {
-            const error = new UserError(ajv.errorsText(validator.errors, { dataVar: targetName }) + '.');
+            const error = new UserError(this[kFormatValidationErrors](validator, targetName));
             if (throwOnErrors) {
                 throw error;
             }
@@ -239,4 +247,7 @@ export class Base extends EventEmitter {
     [kInspect](...args) {
         debugDump(`client:${this[kInstance]}`, ...args);
     }
+    [kFormatValidationErrors](validator, targetName) {
+        return ajv.errorsText(validator.errors, { dataVar: '$dataVar$' }).replaceAll('$dataVar$', targetName) + '.';
+    }
 }

package/dist/clients/base/options.d.ts

@@ -77,6 +77,10 @@ export declare const baseOptionsSchema: {
         strict: {
             type: string;
         };
+        metrics: {
+            type: string;
+            additionalProperties: boolean;
+        };
     };
     required: string[];
     additionalProperties: boolean;

package/dist/clients/base/options.js

@@ -28,7 +28,8 @@ export const baseOptionsSchema = {
         maxInflights: { type: 'number', minimum: 0 },
         metadataMaxAge: { type: 'number', minimum: 0 },
         autocreateTopics: { type: 'boolean' },
-        strict: { type: 'boolean' }
+        strict: { type: 'boolean' },
+        metrics: { type: 'object', additionalProperties: true }
     },
     required: ['clientId', 'bootstrapBrokers'],
     additionalProperties: true

package/dist/clients/base/types.d.ts

@@ -1,4 +1,5 @@
 import { type Broker, type ConnectionOptions } from '../../network/connection.ts';
+import { type Metrics } from '../metrics.ts';
 export interface TopicWithPartitionAndOffset {
     topic: string;
     partition: number;
@@ -29,6 +30,7 @@ export interface BaseOptions extends ConnectionOptions {
     metadataMaxAge?: number;
     autocreateTopics?: boolean;
     strict?: boolean;
+    metrics?: Metrics;
 }
 export interface MetadataOptions {
     topics: string[];

package/dist/clients/consumer/consumer.d.ts

@@ -1,5 +1,6 @@
 import { type FetchResponse } from '../../apis/consumer/fetch.ts';
-import { Base } from '../base/base.ts';
+import { type ConnectionPool } from '../../network/connection-pool.ts';
+import { Base, kFetchConnections } from '../base/base.ts';
 import { type CallbackWithPromise } from '../callbacks.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
@@ -11,7 +12,9 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     memberId: string | null;
     topics: TopicsMap;
     assignments: GroupAssignment[] | null;
+    [kFetchConnections]: ConnectionPool;
     constructor(options: ConsumerOptions<Key, Value, HeaderKey, HeaderValue>);
+    get streamsCount(): number;
     close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     close(force?: boolean): Promise<void>;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>): void;
@@ -28,6 +31,6 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     findGroupCoordinator(): Promise<number>;
     joinGroup(options: GroupOptions, callback: CallbackWithPromise<string>): void;
     joinGroup(options: GroupOptions): Promise<string>;
-    leaveGroup(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
+    leaveGroup(force?: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     leaveGroup(force?: boolean): Promise<void>;
 }

package/dist/clients/consumer/consumer.js

@@ -11,11 +11,12 @@ import { api as findCoordinatorV6 } from "../../apis/metadata/find-coordinator.j
 import { UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Writer } from "../../protocol/writer.js";
-import { Base, kBootstrapBrokers, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
+import { Base, kBootstrapBrokers, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { defaultBaseOptions } from "../base/options.js";
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../callbacks.js";
+import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
-import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
+import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
 import { TopicsMap } from "./topics-map.js";
 export class Consumer extends Base {
     groupId;
@@ -44,7 +45,9 @@ export class Consumer extends Base {
   
       In order to avoid consumer group problems, we separate FetchRequest only on a separate connection.
     */
-    #fetchConnectionPool;
+    [kFetchConnections];
+    // Metrics
+    #metricActiveStreams;
     constructor(options) {
         super(options);
         this[kOptions] = Object.assign({}, defaultBaseOptions, defaultConsumerOptions, options);
@@ -61,9 +64,17 @@ export class Consumer extends Base {
         this.#coordinatorId = null;
         this.#heartbeatInterval = null;
         this.#streams = new Set();
-        this.#validateGroupOptions(this[kOptions]);
+        this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
         // Initialize connection pool
-        this.#fetchConnectionPool = this[kCreateConnectionPool]();
+        this[kFetchConnections] = this[kCreateConnectionPool]();
+        if (this[kPrometheus]) {
+            ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').inc();
+            this.#metricActiveStreams = ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers_streams', 'Number of active Kafka consumers streams');
+            this.topics.setMetric(ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers_topics', 'Number of topics being consumed'));
+        }
+    }
+    get streamsCount() {
+        return this.#streams.size;
     }
     close(force, callback) {
         if (typeof force === 'function') {
@@ -80,20 +91,33 @@ export class Consumer extends Base {
         this[kClosed] = true;
         const closer = this.#membershipActive
             ? this.#leaveGroup.bind(this)
-            : function (_, callback) {
+            : function noopCloser(_, callback) {
                 callback(null);
             };
         closer(force, error => {
             if (error) {
+                this[kClosed] = false;
                 callback(error);
                 return;
             }
-            this.#fetchConnectionPool.close(error => {
+            this[kFetchConnections].close(error => {
                 if (error) {
+                    this[kClosed] = false;
                     callback(error);
                     return;
                 }
-                super.close(callback);
+                super.close(error => {
+                    if (error) {
+                        this[kClosed] = false;
+                        callback(error);
+                        return;
+                    }
+                    this.topics.clear();
+                    if (this[kPrometheus]) {
+                        ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').dec();
+                    }
+                    callback(null);
+                });
             });
         });
         return callback[kCallbackPromise];
@@ -224,15 +248,24 @@ export class Consumer extends Base {
             return callback[kCallbackPromise];
         }
         this.#membershipActive = false;
-        this.#leaveGroup(force, callback);
+        this.#leaveGroup(force, error => {
+            if (error) {
+                this.#membershipActive = true;
+                callback(error);
+                return;
+            }
+            callback(null);
+        });
         return callback[kCallbackPromise];
     }
-    #consume(options, callback) {
+    #consume(options, callback, trackTopics = true) {
         // Subscribe all topics
         let joinNeeded = this.memberId === null;
-        for (const topic of options.topics) {
-            if (this.topics.track(topic)) {
-                joinNeeded = true;
+        if (trackTopics) {
+            for (const topic of options.topics) {
+                if (this.topics.track(topic)) {
+                    joinNeeded = true;
+                }
             }
         }
         // If we need to (re)join the group, do that first and then try again
@@ -242,14 +275,16 @@ export class Consumer extends Base {
                     callback(error, undefined);
                     return;
                 }
-                this.#consume(options, callback);
+                this.#consume(options, callback, false);
             });
             return callback[kCallbackPromise];
         }
         // Create the stream and start consuming
         const stream = new MessagesStream(this, options);
         this.#streams.add(stream);
+        this.#metricActiveStreams?.inc();
         stream.once('close', () => {
+            this.#metricActiveStreams?.dec();
             this.#streams.delete(stream);
         });
         callback(null, stream);
@@ -267,7 +302,7 @@ export class Consumer extends Base {
                     retryCallback(new UserError(`Cannot find broker with node id ${options.node}`), undefined);
                     return;
                 }
-                this.#fetchConnectionPool.get(broker, (error, connection) => {
+                this[kFetchConnections].get(broker, (error, connection) => {
                     if (error) {
                         retryCallback(error, undefined);
                         return;
@@ -522,12 +557,15 @@ export class Consumer extends Base {
                 memberId: this.memberId,
                 generationId: this.generationId
             });
+            this.memberId = null;
+            this.generationId = 0;
+            this.assignments = null;
             callback(null);
         });
     }
     #syncGroup(assignments, callback) {
         if (!this.#membershipActive) {
-            callback(null, undefined);
+            callback(null, []);
             return;
         }
         if (!Array.isArray(assignments)) {
@@ -549,10 +587,6 @@ export class Consumer extends Base {
                         callback(error, undefined);
                         return;
                     }
-                    if (!this.#membershipActive) {
-                        callback(null, undefined);
-                        return;
-                    }
                     this.#syncGroup(this.#roundRobinAssignments(metadata), callback);
                 });
                 return;
@@ -573,10 +607,6 @@ export class Consumer extends Base {
                 callback(error, undefined);
                 return;
             }
-            else if (response.assignment.length === 0) {
-                callback(null, []);
-                return;
-            }
             // Read the assignment back
             const reader = Reader.from(response.assignment);
             const assignments = reader.readArray(r => {
@@ -593,13 +623,15 @@ export class Consumer extends Base {
         this.#performDeduplicateGroupOperaton('heartbeat', (connection, groupCallback) => {
             // We have left the group in the meanwhile, abort
             if (!this.#membershipActive) {
+                this.emitWithDebug('consumer:heartbeat', 'cancel', eventPayload);
                 return;
             }
-            this.emitWithDebug('consumer:heartbeat', 'start');
+            this.emitWithDebug('consumer:heartbeat', 'start', eventPayload);
             heartbeatV4(connection, this.groupId, this.generationId, this.memberId, null, groupCallback);
         }, error => {
             // The heartbeat has been aborted elsewhere, ignore the response
             if (this.#heartbeatInterval === null || !this.#membershipActive) {
+                this.emitWithDebug('consumer:heartbeat', 'cancel', eventPayload);
                 return;
             }
             if (error) {
@@ -656,15 +688,11 @@ export class Consumer extends Base {
             });
         });
     }
-    #validateGroupOptions(options) {
-        if (options.rebalanceTimeout < options.sessionTimeout) {
-            throw new UserError('/options/rebalanceTimeout must be greater than or equal to /options/sessionTimeout.');
-        }
-        if (options.heartbeatInterval > options.sessionTimeout) {
-            throw new UserError('/options/heartbeatInterval must be less than or equal to /options/sessionTimeout.');
-        }
-        if (options.heartbeatInterval > options.rebalanceTimeout) {
-            throw new UserError('/options/heartbeatInterval must be less than or equal to /options/rebalanceTimeout.');
+    #validateGroupOptions(options, validator) {
+        validator ??= groupOptionsValidator;
+        const valid = validator(options);
+        if (!valid) {
+            throw new UserError(this[kFormatValidationErrors](validator, '/options'));
         }
     }
     /*
@@ -762,6 +790,10 @@ export class Consumer extends Base {
         else if (protocolError.memberId && !this.memberId) {
             this.memberId = protocolError.memberId;
         }
+        // This is only used in testing
+        if (protocolError.cancelMembership) {
+            this.#membershipActive = false;
+        }
         return protocolError;
     }
 }

package/dist/clients/consumer/messages-stream.js

@@ -1,8 +1,9 @@
 import { Readable } from 'node:stream';
 import { ListOffsetTimestamps } from "../../apis/enumerations.js";
 import { UserError } from "../../errors.js";
-import { kInspect } from "../base/base.js";
+import { kInspect, kPrometheus } from "../base/base.js";
 import { createPromisifiedCallback, kCallbackPromise, noopCallback } from "../callbacks.js";
+import { ensureMetric } from "../metrics.js";
 import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
@@ -28,6 +29,7 @@ export class MessagesStream extends Readable {
     #autocommitInflight;
     #shouldClose;
     #closeCallbacks;
+    #metricsConsumedMessages;
     constructor(consumer, options) {
         const { autocommit, mode, fallbackMode, offsets, deserializers, ..._options } = options;
         if (offsets && mode !== MessagesStreamModes.MANUAL) {
@@ -82,6 +84,9 @@ export class MessagesStream extends Readable {
                 this.#fetch();
             });
         });
+        if (consumer[kPrometheus]) {
+            this.#metricsConsumedMessages = ensureMetric(consumer[kPrometheus], 'Counter', 'kafka_consumers_messages', 'Number of consumed Kafka messages');
+        }
     }
     close(callback) {
         if (!callback) {
@@ -270,6 +275,7 @@ export class MessagesStream extends Readable {
                     for (const [headerKey, headerValue] of record.headers) {
                         headers.set(headerKeyDeserializer(headerKey), headerValueDeserializer(headerValue));
                     }
+                    this.#metricsConsumedMessages?.inc();
                     canPush = this.push({
                         key,
                         value,
@@ -327,6 +333,10 @@ export class MessagesStream extends Readable {
         });
     }
     #refreshOffsets(callback) {
+        if (this.#topics.length === 0) {
+            callback(null);
+            return;
+        }
         // List topic offsets
         this.#consumer.listOffsets({
             topics: this.#topics,

package/dist/clients/consumer/options.d.ts

@@ -1,4 +1,120 @@
 import { type ConsumerOptions } from './types.ts';
+export declare const groupOptionsProperties: {
+    sessionTimeout: {
+        type: string;
+        minimum: number;
+    };
+    rebalanceTimeout: {
+        type: string;
+        minimum: number;
+    };
+    heartbeatInterval: {
+        type: string;
+        minimum: number;
+    };
+    protocols: {
+        type: string;
+        items: {
+            type: string;
+            properties: {
+                name: {
+                    type: string;
+                    pattern: string;
+                };
+                version: {
+                    type: string;
+                    minimum: number;
+                };
+                topics: {
+                    type: string;
+                    items: {
+                        type: string;
+                    };
+                };
+                metadata: {
+                    oneOf: ({
+                        type: string;
+                        buffer?: undefined;
+                    } | {
+                        buffer: boolean;
+                        type?: undefined;
+                    })[];
+                };
+            };
+        };
+    };
+};
+export declare const groupOptionsAdditionalValidations: {
+    rebalanceTimeout: {
+        properties: {
+            rebalanceTimeout: {
+                type: string;
+                minimum: number;
+                gteProperty: string;
+            };
+        };
+    };
+    heartbeatInterval: {
+        properties: {
+            heartbeatInterval: {
+                type: string;
+                minimum: number;
+                allOf: {
+                    lteProperty: string;
+                }[];
+            };
+        };
+    };
+};
+export declare const consumeOptionsProperties: {
+    autocommit: {
+        oneOf: ({
+            type: string;
+            minimum?: undefined;
+        } | {
+            type: string;
+            minimum: number;
+        })[];
+    };
+    minBytes: {
+        type: string;
+        minimum: number;
+    };
+    maxBytes: {
+        type: string;
+        minimum: number;
+    };
+    maxWaitTime: {
+        type: string;
+        minimum: number;
+    };
+    isolationLevel: {
+        type: string;
+        enum: string[];
+    };
+    deserializers: {
+        type: string;
+        properties: {
+            key: {
+                function: boolean;
+            };
+            value: {
+                function: boolean;
+            };
+            headerKey: {
+                function: boolean;
+            };
+            headerValue: {
+                function: boolean;
+            };
+        };
+        additionalProperties: boolean;
+    };
+    highWaterMark: {
+        type: string;
+        minimum: number;
+    };
+};
 export declare const groupOptionsSchema: {
     type: string;
     properties: {
@@ -500,6 +616,9 @@ export declare const listOffsetsOptionsSchema: {
     additionalProperties: boolean;
 };
 export declare const groupOptionsValidator: import("ajv").ValidateFunction<unknown>;
+export declare const groupIdAndOptionsValidator: import("ajv").ValidateFunction<{
+    groupId: any;
+}>;
 export declare const consumeOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;

package/dist/clients/consumer/options.js

@@ -3,7 +3,7 @@ import { ajv } from "../../utils.js";
 import { idProperty, topicWithPartitionAndOffsetProperties } from "../base/options.js";
 import { serdeProperties } from "../serde.js";
 import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
-const groupOptionsProperties = {
+export const groupOptionsProperties = {
     sessionTimeout: { type: 'number', minimum: 0 },
     rebalanceTimeout: { type: 'number', minimum: 0 },
     heartbeatInterval: { type: 'number', minimum: 0 },
@@ -23,7 +23,34 @@ const groupOptionsProperties = {
         }
     }
 };
-const consumeOptionsProperties = {
+export const groupOptionsAdditionalValidations = {
+    rebalanceTimeout: {
+        properties: {
+            rebalanceTimeout: {
+                type: 'number',
+                minimum: 0,
+                gteProperty: 'sessionTimeout'
+            }
+        }
+    },
+    heartbeatInterval: {
+        properties: {
+            heartbeatInterval: {
+                type: 'number',
+                minimum: 0,
+                allOf: [
+                    {
+                        lteProperty: 'sessionTimeout'
+                    },
+                    {
+                        lteProperty: 'rebalanceTimeout'
+                    }
+                ]
+            }
+        }
+    }
+};
+export const consumeOptionsProperties = {
     autocommit: { oneOf: [{ type: 'boolean' }, { type: 'number', minimum: 100 }] },
     minBytes: { type: 'number', minimum: 0 },
     maxBytes: { type: 'number', minimum: 0 },
@@ -156,7 +183,20 @@ export const listOffsetsOptionsSchema = {
     required: ['topics'],
     additionalProperties: false
 };
-export const groupOptionsValidator = ajv.compile(groupOptionsSchema);
+export const groupOptionsValidator = ajv.compile({
+    ...groupOptionsSchema,
+    dependentSchemas: groupOptionsAdditionalValidations
+});
+export const groupIdAndOptionsValidator = ajv.compile({
+    type: 'object',
+    properties: {
+        groupId: idProperty,
+        ...groupOptionsProperties
+    },
+    required: ['groupId'],
+    additionalProperties: true,
+    dependentSchemas: groupOptionsAdditionalValidations
+});
 export const consumeOptionsValidator = ajv.compile(consumeOptionsSchema);
 export const consumerOptionsValidator = ajv.compile(consumerOptionsSchema);
 export const fetchOptionsValidator = ajv.compile(fetchOptionsSchema);

package/dist/clients/consumer/topics-map.d.ts

@@ -1,8 +1,11 @@
+import { type Gauge } from '../metrics.ts';
 export declare class TopicsMap extends Map<string, number> {
     #private;
     get current(): string[];
+    clear(): void;
     track(topic: string): boolean;
     trackAll(...topics: string[]): boolean[];
     untrack(topic: string): boolean;
     untrackAll(...topics: string[]): boolean[];
+    setMetric(metric: Gauge): void;
 }

package/dist/clients/consumer/topics-map.js

@@ -1,8 +1,15 @@
 export class TopicsMap extends Map {
     #current = [];
+    #metric;
     get current() {
         return this.#current;
     }
+    clear() {
+        for (const k of this.keys()) {
+            this.untrack(k);
+        }
+        super.clear();
+    }
     track(topic) {
         let updated = false;
         let existing = this.get(topic);
@@ -11,6 +18,9 @@ export class TopicsMap extends Map {
             updated = true;
         }
         this.set(topic, existing + 1);
+        if (existing === 0) {
+            this.#metric?.inc();
+        }
         if (updated) {
             this.#updateCurrentList();
         }
@@ -28,6 +38,7 @@ export class TopicsMap extends Map {
         if (existing === 1) {
             this.delete(topic);
             this.#updateCurrentList();
+            this.#metric?.dec();
             return true;
         }
         else if (typeof existing === 'number') {
@@ -42,6 +53,9 @@ export class TopicsMap extends Map {
         }
         return updated;
     }
+    setMetric(metric) {
+        this.#metric = metric;
+    }
     #updateCurrentList() {
         this.#current = Array.from(this.keys());
     }

package/dist/clients/metrics.d.ts

@@ -0,0 +1,54 @@
+type RegistryContentType = 'application/openmetrics-text; version=1.0.0; charset=utf-8' | 'text/plain; version=0.0.4; charset=utf-8';
+export interface Metric {
+    name?: string;
+    get(): Promise<unknown>;
+    reset: () => void;
+    labels(labels: any): any;
+}
+export interface Counter extends Metric {
+    inc: (value?: number) => void;
+}
+export interface Gauge extends Metric {
+    inc: (value?: number) => void;
+    dec: (value?: number) => void;
+}
+export interface Histogram {
+}
+export interface Summary {
+}
+export interface Registry {
+    getSingleMetric: (name: string) => Counter | Gauge | any;
+    metrics(): Promise<string>;
+    clear(): void;
+    resetMetrics(): void;
+    registerMetric(metric: Metric): void;
+    getMetricsAsJSON(): Promise<any>;
+    getMetricsAsArray(): any[];
+    removeSingleMetric(name: string): void;
+    setDefaultLabels(labels: object): void;
+    getSingleMetricAsString(name: string): Promise<string>;
+    readonly contentType: RegistryContentType;
+    setContentType(contentType: RegistryContentType): void;
+}
+export interface Prometheus {
+    Counter: new (options: {
+        name: string;
+        help: string;
+        registers: Registry[];
+        labelNames?: string[];
+    }) => Counter;
+    Gauge: new (options: {
+        name: string;
+        help: string;
+        registers: Registry[];
+        labelNames?: string[];
+    }) => Gauge;
+    Registry: new (contentType?: string) => Registry;
+}
+export interface Metrics {
+    registry: Registry;
+    client: Prometheus;
+    labels?: Record<string, any>;
+}
+export declare function ensureMetric<MetricType extends Metric>(metrics: Metrics, type: 'Gauge' | 'Counter', name: string, help: string): MetricType;
+export {};

package/dist/clients/metrics.js

@@ -0,0 +1,18 @@
+// Interfaces to make the package compatible with prom-client
+export function ensureMetric(metrics, type, name, help) {
+    let metric = metrics.registry.getSingleMetric(name);
+    const labels = Object.keys(metrics.labels ?? {});
+    if (!metric) {
+        metric = new metrics.client[type]({
+            name,
+            help,
+            registers: [metrics.registry],
+            labelNames: labels
+        });
+    }
+    else {
+        // @ts-expect-error Overriding internal API
+        metric.labelNames = metric.sortedLabelNames = Array.from(new Set([...metric.labelNames, ...labels])).sort();
+    }
+    return metric.labels(metrics.labels ?? {});
+}

package/dist/clients/producer/options.d.ts

@@ -1,3 +1,33 @@
+export declare const produceOptionsProperties: {
+    producerId: {
+        bigint: boolean;
+    };
+    producerEpoch: {
+        type: string;
+    };
+    idempotent: {
+        type: string;
+    };
+    acks: {
+        type: string;
+        enum: (0 | 1 | -1)[];
+        errorMessage: string;
+    };
+    compression: {
+        type: string;
+        enum: string[];
+        errorMessage: string;
+    };
+    partitioner: {
+        function: boolean;
+    };
+    autocreateTopics: {
+        type: string;
+    };
+    repeatOnStaleMetadata: {
+        type: string;
+    };
+};
 export declare const produceOptionsSchema: {
     type: string;
     properties: {

package/dist/clients/producer/options.js

@@ -3,7 +3,7 @@ import { compressionsAlgorithms } from "../../protocol/compression.js";
 import { messageSchema } from "../../protocol/records.js";
 import { ajv, enumErrorMessage } from "../../utils.js";
 import { serdeProperties } from "../serde.js";
-const produceOptionsProperties = {
+export const produceOptionsProperties = {
     producerId: { bigint: true },
     producerEpoch: { type: 'number' },
     idempotent: { type: 'boolean' },

package/dist/clients/producer/producer.d.ts

@@ -6,6 +6,8 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     constructor(options: ProducerOptions<Key, Value, HeaderKey, HeaderValue>);
     get producerId(): bigint | undefined;
     get producerEpoch(): number | undefined;
+    close(callback: CallbackWithPromise<void>): void;
+    close(): Promise<void>;
     initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<ProducerInfo>): void;
     initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<ProducerInfo>;
     send(options: SendOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<ProduceResult>): void;

package/dist/clients/producer/producer.js

@@ -4,8 +4,9 @@ import { api as produceV11 } from "../../apis/producer/produce.js";
 import { UserError } from "../../errors.js";
 import { murmur2 } from "../../protocol/murmur2.js";
 import { NumericMap } from "../../utils.js";
-import { Base, kBootstrapBrokers, kCheckNotClosed, kClearMetadata, kConnections, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
+import { Base, kBootstrapBrokers, kCheckNotClosed, kClearMetadata, kClosed, kConnections, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../callbacks.js";
+import { ensureMetric } from "../metrics.js";
 import { produceOptionsValidator, producerOptionsValidator, sendOptionsValidator } from "./options.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
@@ -22,6 +23,7 @@ export class Producer extends Base {
     #valueSerializer;
     #headerKeySerializer;
     #headerValueSerializer;
+    #metricsProducedMessages;
     constructor(options) {
         if (options.idempotent) {
             options.maxInflights = 1;
@@ -37,6 +39,10 @@ export class Producer extends Base {
         this.#headerKeySerializer = options.serializers?.headerKey ?? noopSerializer;
         this.#headerValueSerializer = options.serializers?.headerValue ?? noopSerializer;
         this[kValidateOptions](options, producerOptionsValidator, '/options');
+        if (this[kPrometheus]) {
+            ensureMetric(this[kPrometheus], 'Gauge', 'kafka_producers', 'Number of active Kafka producers').inc();
+            this.#metricsProducedMessages = ensureMetric(this[kPrometheus], 'Counter', 'kafka_produced_messages', 'Number of produced Kafka messages');
+        }
     }
     get producerId() {
         return this.#producerInfo?.producerId;
@@ -44,6 +50,28 @@ export class Producer extends Base {
     get producerEpoch() {
         return this.#producerInfo?.producerEpoch;
     }
+    close(callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kClosed]) {
+            callback(null);
+            return callback[kCallbackPromise];
+        }
+        this[kClosed] = true;
+        super.close(error => {
+            if (error) {
+                this[kClosed] = false;
+                callback(error);
+                return;
+            }
+            if (this[kPrometheus]) {
+                ensureMetric(this[kPrometheus], 'Gauge', 'kafka_producers', 'Number of active Kafka producers').dec();
+            }
+            callback(null);
+        });
+        return callback[kCallbackPromise];
+    }
     initIdempotentProducer(options, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -194,14 +222,15 @@ export class Producer extends Base {
             }
             // Track nodes so that we can get their ID for delayed write reporting
             const nodes = [];
-            runConcurrentCallbacks('Producing messages failed.', messagesByDestination, ([destination, messages], concurrentCallback) => {
+            runConcurrentCallbacks('Producing messages failed.', messagesByDestination, ([destination, destinationMessages], concurrentCallback) => {
                 nodes.push(destination);
-                this.#performSingleDestinationSend(topics, messages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, concurrentCallback);
+                this.#performSingleDestinationSend(topics, destinationMessages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, concurrentCallback);
             }, (error, apiResults) => {
                 if (error) {
                     callback(error, undefined);
                     return;
                 }
+                this.#metricsProducedMessages?.inc(messages.length);
                 const results = {};
                 if (sendOptions.acks === ProduceAcks.NO_RESPONSE) {
                     const unwritableNodes = [];

package/dist/utils.d.ts

@@ -1,9 +1,14 @@
-import { Ajv } from 'ajv';
+import { Ajv2020 } from 'ajv/dist/2020.js';
 import debug from 'debug';
 import { type DynamicBuffer } from './protocol/dynamic-buffer.ts';
+export interface DataValidationContext {
+    parentData: {
+        [k: string | number]: any;
+    };
+}
 export type DebugDumpLogger = (...args: any[]) => void;
 export { setTimeout as sleep } from 'node:timers/promises';
-export declare const ajv: Ajv;
+export declare const ajv: Ajv2020;
 export declare const loggers: Record<string, debug.Debugger>;
 export declare class NumericMap extends Map<string, number> {
     getWithDefault(key: string, fallback: number): number;

package/dist/utils.js

@@ -1,11 +1,11 @@
 import { Unpromise } from '@watchable/unpromise';
-import { Ajv } from 'ajv';
 import ajvErrors from 'ajv-errors';
+import { Ajv2020 } from 'ajv/dist/2020.js';
+import debug from 'debug';
 import { setTimeout as sleep } from 'node:timers/promises';
 import { inspect } from 'node:util';
-import debug from 'debug';
 export { setTimeout as sleep } from 'node:timers/promises';
-export const ajv = new Ajv({ allErrors: true, coerceTypes: false, strict: true });
+export const ajv = new Ajv2020({ allErrors: true, coerceTypes: false, strict: true });
 export const loggers = {
     protocol: debug('plt:kafka:protocol'),
     client: debug('plt:kafka:client'),
@@ -53,6 +53,30 @@ ajv.addKeyword({
         message: 'must be Buffer'
     }
 });
+ajv.addKeyword({
+    keyword: 'gteProperty',
+    validate(property, current, _, context) {
+        const root = context?.parentData;
+        return current >= root[property];
+    },
+    error: {
+        message({ schema }) {
+            return `must be greater than or equal to $dataVar$/${schema}`;
+        }
+    }
+});
+ajv.addKeyword({
+    keyword: 'lteProperty',
+    validate(property, current, _, context) {
+        const root = context?.parentData;
+        return current < root[property];
+    },
+    error: {
+        message({ schema }) {
+            return `must be less than or equal to $dataVar$/${schema}`;
+        }
+    }
+});
 export class NumericMap extends Map {
     getWithDefault(key, fallback) {
         return this.get(key) ?? fallback;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -24,20 +24,6 @@
   "type": "module",
   "exports": "./dist/index.js",
   "types": "./dist/index.d.ts",
-  "scripts": {
-    "build": "rm -rf dist && tsc -p tsconfig.base.json",
-    "lint": "eslint --cache",
-    "typecheck": "tsc -p . --noEmit",
-    "format": "prettier -w benchmarks playground src test",
-    "test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --no-warnings --test --test-timeout=180000 'test/**/*.test.ts'",
-    "test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --no-warnings --test --test-timeout=180000 'test/**/*.test.ts'",
-    "ci": "npm run build && npm run lint && npm run test:ci",
-    "prepublishOnly": "npm run ci",
-    "postpublish": "git push origin && git push origin -f --tags",
-    "generate:apis": "node --experimental-strip-types scripts/generate-apis.ts",
-    "generate:errors": "node --experimental-strip-types scripts/generate-errors.ts",
-    "create:api": "node --experimental-strip-types scripts/create-api.ts"
-  },
   "dependencies": {
     "@watchable/unpromise": "^1.0.2",
     "ajv": "^8.17.1",
@@ -53,23 +39,36 @@
   },
   "devDependencies": {
     "@platformatic/rdkafka": "^4.0.0",
-    "cronometro": "^5.3.0",
-    "node-rdkafka": "^3.3.1",
-    "kafkajs": "^2.2.4",
     "@types/debug": "^4.1.12",
     "@types/node": "^22.13.5",
     "c8": "^10.1.3",
-    "cleaner-spec-reporter": "^0.3.1",
+    "cleaner-spec-reporter": "^0.4.0",
+    "cronometro": "^5.3.0",
     "eslint": "^9.21.0",
     "hwp": "^0.4.1",
     "json5": "^2.2.3",
+    "kafkajs": "^2.2.4",
     "neostandard": "^0.12.1",
+    "node-rdkafka": "^3.3.1",
     "parse5": "^7.2.1",
     "prettier": "^3.5.3",
+    "prom-client": "^15.1.3",
     "scule": "^1.3.0",
     "typescript": "^5.7.3"
   },
   "engines": {
     "node": ">= 22.14.0"
+  },
+  "scripts": {
+    "build": "rm -rf dist && tsc -p tsconfig.base.json",
+    "lint": "eslint --cache",
+    "typecheck": "tsc -p . --noEmit",
+    "format": "prettier -w benchmarks playground src test",
+    "test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
+    "test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
+    "ci": "npm run build && npm run lint && npm run test:ci",
+    "generate:apis": "node --experimental-strip-types scripts/generate-apis.ts",
+    "generate:errors": "node --experimental-strip-types scripts/generate-errors.ts",
+    "create:api": "node --experimental-strip-types scripts/create-api.ts"
   }
-}
+}