@platformatic/kafka 1.0.0 → 1.2.0

package/dist/clients/base/base.d.ts

@@ -1,18 +1,24 @@
 import { type ValidateFunction } from 'ajv';
 import { EventEmitter } from 'node:events';
-import { type Callback } from '../../apis/definitions.ts';
+import { type API, type Callback } from '../../apis/definitions.ts';
+import { type ApiVersionsResponseApi } from '../../apis/metadata/api-versions-v3.ts';
+import { type ClientType } from '../../diagnostic.ts';
 import { ConnectionPool } from '../../network/connection-pool.ts';
 import { type Broker } from '../../network/connection.ts';
+import { kInstance } from '../../symbols.ts';
 import { type CallbackWithPromise } from '../callbacks.ts';
 import { type Metrics } from '../metrics.ts';
 import { type BaseOptions, type ClusterMetadata, type MetadataOptions } from './types.ts';
 export declare const kClientId: unique symbol;
 export declare const kBootstrapBrokers: unique symbol;
+export declare const kApis: unique symbol;
+export declare const kGetApi: unique symbol;
 export declare const kOptions: unique symbol;
 export declare const kConnections: unique symbol;
 export declare const kFetchConnections: unique symbol;
 export declare const kCreateConnectionPool: unique symbol;
 export declare const kClosed: unique symbol;
+export declare const kListApis: unique symbol;
 export declare const kMetadata: unique symbol;
 export declare const kCheckNotClosed: unique symbol;
 export declare const kClearMetadata: unique symbol;
@@ -22,33 +28,43 @@ export declare const kPerformDeduplicated: unique symbol;
 export declare const kValidateOptions: unique symbol;
 export declare const kInspect: unique symbol;
 export declare const kFormatValidationErrors: unique symbol;
-export declare const kInstance: unique symbol;
 export declare const kPrometheus: unique symbol;
-export declare class Base<OptionsType extends BaseOptions> extends EventEmitter {
+export declare const kClientType: unique symbol;
+export declare const kAfterCreate: unique symbol;
+export declare class Base<OptionsType extends BaseOptions = BaseOptions> extends EventEmitter {
     #private;
     [kInstance]: number;
     [kClientId]: string;
+    [kClientType]: ClientType;
     [kBootstrapBrokers]: Broker[];
+    [kApis]: ApiVersionsResponseApi[];
     [kOptions]: OptionsType;
     [kConnections]: ConnectionPool;
     [kClosed]: boolean;
     [kPrometheus]: Metrics | undefined;
     constructor(options: OptionsType);
+    get instanceId(): number;
     get clientId(): string;
     get closed(): boolean;
+    get type(): ClientType;
     emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
+    listApis(callback: CallbackWithPromise<ApiVersionsResponseApi[]>): void;
+    listApis(): Promise<ApiVersionsResponseApi[]>;
     metadata(options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     metadata(options: MetadataOptions): Promise<ClusterMetadata>;
     [kCreateConnectionPool](): ConnectionPool;
+    [kListApis](callback: CallbackWithPromise<ApiVersionsResponseApi[]>): void;
     [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
     [kClearMetadata](): void;
     [kParseBroker](broker: Broker | string): Broker;
     [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
     [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
+    [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
     [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
     [kInspect](...args: unknown[]): void;
     [kFormatValidationErrors](validator: ValidateFunction<unknown>, targetName: string): string;
+    [kAfterCreate](type: ClientType): void;
 }

package/dist/clients/base/base.js

@@ -1,17 +1,23 @@
 import { EventEmitter } from 'node:events';
-import { api as metadataV12 } from "../../apis/metadata/metadata.js";
-import { MultipleErrors, NetworkError, UserError } from "../../errors.js";
+import * as apis from "../../apis/index.js";
+import { api as apiVersionsV3 } from "../../apis/metadata/api-versions-v3.js";
+import { baseApisChannel, baseMetadataChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
+import { MultipleErrors, NetworkError, UnsupportedApiError, UserError } from "../../errors.js";
 import { ConnectionPool } from "../../network/connection-pool.js";
+import { kInstance } from "../../symbols.js";
 import { ajv, debugDump, loggers } from "../../utils.js";
 import { createPromisifiedCallback, kCallbackPromise } from "../callbacks.js";
-import { baseOptionsValidator, defaultBaseOptions, defaultPort, metadataOptionsValidator } from "./options.js";
+import { baseOptionsValidator, clientSoftwareName, clientSoftwareVersion, defaultBaseOptions, defaultPort, metadataOptionsValidator } from "./options.js";
 export const kClientId = Symbol('plt.kafka.base.clientId');
 export const kBootstrapBrokers = Symbol('plt.kafka.base.bootstrapBrokers');
+export const kApis = Symbol('plt.kafka.base.apis');
+export const kGetApi = Symbol('plt.kafka.base.getApi');
 export const kOptions = Symbol('plt.kafka.base.options');
 export const kConnections = Symbol('plt.kafka.base.connections');
 export const kFetchConnections = Symbol('plt.kafka.base.fetchCnnections');
 export const kCreateConnectionPool = Symbol('plt.kafka.base.createConnectionPool');
 export const kClosed = Symbol('plt.kafka.base.closed');
+export const kListApis = Symbol('plt.kafka.base.listApis');
 export const kMetadata = Symbol('plt.kafka.base.metadata');
 export const kCheckNotClosed = Symbol('plt.kafka.base.checkNotClosed');
 export const kClearMetadata = Symbol('plt.kafka.base.clearMetadata');
@@ -21,15 +27,18 @@ export const kPerformDeduplicated = Symbol('plt.kafka.base.performDeduplicated')
 export const kValidateOptions = Symbol('plt.kafka.base.validateOptions');
 export const kInspect = Symbol('plt.kafka.base.inspect');
 export const kFormatValidationErrors = Symbol('plt.kafka.base.formatValidationErrors');
-export const kInstance = Symbol('plt.kafka.base.instance');
 export const kPrometheus = Symbol('plt.kafka.base.prometheus');
+export const kClientType = Symbol('plt.kafka.base.clientType');
+export const kAfterCreate = Symbol('plt.kafka.base.afterCreate');
 let currentInstance = 0;
 export class Base extends EventEmitter {
-    // This is just used for debugging
+    // This is declared using a symbol (a.k.a protected/friend) to make it available in ConnectionPool and MessagesStream
     [kInstance];
     // General status - Use symbols rather than JS private property to make them "protected" as in C++
     [kClientId];
+    [kClientType];
     [kBootstrapBrokers];
+    [kApis];
     [kOptions];
     [kConnections];
     [kClosed];
@@ -38,7 +47,9 @@ export class Base extends EventEmitter {
     #inflightDeduplications;
     constructor(options) {
         super();
+        this[kClientType] = 'base';
         this[kInstance] = currentInstance++;
+        this[kApis] = [];
         // Validate options
         this[kOptions] = Object.assign({}, defaultBaseOptions, options);
         this[kValidateOptions](this[kOptions], baseOptionsValidator, '/options');
@@ -57,19 +68,22 @@ export class Base extends EventEmitter {
             this[kPrometheus] = options.metrics;
         }
     }
-    /* c8 ignore next 3 */
+    get instanceId() {
+        return this[kInstance];
+    }
     get clientId() {
         return this[kClientId];
     }
-    /* c8 ignore next 3 */
     get closed() {
         return this[kClosed] === true;
     }
+    get type() {
+        return this[kClientType];
+    }
     emitWithDebug(section, name, ...args) {
         if (!section) {
             return this.emit(name, ...args);
         }
-        /* c8 ignore next */
         loggers[section]?.({ event: name, payload: args });
         return this.emit(`${section}:${name}`, ...args);
     }
@@ -81,6 +95,13 @@ export class Base extends EventEmitter {
         this[kConnections].close(callback);
         return callback[kCallbackPromise];
     }
+    listApis(callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        baseApisChannel.traceCallback(this[kListApis], 0, createDiagnosticContext({ client: this, operation: 'listApis' }), this, callback);
+        return callback[kCallbackPromise];
+    }
     metadata(options, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -90,7 +111,7 @@ export class Base extends EventEmitter {
             callback(validationError, undefined);
             return callback[kCallbackPromise];
         }
-        this[kMetadata](options, callback);
+        baseMetadataChannel.traceCallback(this[kMetadata], 1, createDiagnosticContext({ client: this, operation: 'metadata' }), this, options, callback);
         return callback[kCallbackPromise];
     }
     [kCreateConnectionPool]() {
@@ -103,6 +124,26 @@ export class Base extends EventEmitter {
         }
         return pool;
     }
+    [kListApis](callback) {
+        this[kPerformDeduplicated]('listApis', deduplicateCallback => {
+            this[kPerformWithRetry]('listApis', retryCallback => {
+                this[kConnections].getFirstAvailable(this[kBootstrapBrokers], (error, connection) => {
+                    if (error) {
+                        retryCallback(error, undefined);
+                        return;
+                    }
+                    // We use V3 to be able to get APIS from Kafka 2.4.0+
+                    apiVersionsV3(connection, clientSoftwareName, clientSoftwareVersion, retryCallback);
+                });
+            }, (error, metadata) => {
+                if (error) {
+                    deduplicateCallback(error, undefined);
+                    return;
+                }
+                deduplicateCallback(null, metadata.apiKeys);
+            }, 0);
+        }, callback);
+    }
     [kMetadata](options, callback) {
         const metadataMaxAge = options.metadataMaxAge ?? this[kOptions].metadataMaxAge;
         const isStale = options.forceUpdate ||
@@ -121,7 +162,13 @@ export class Base extends EventEmitter {
                         retryCallback(error, undefined);
                         return;
                     }
-                    metadataV12(connection, options.topics, autocreateTopics, true, retryCallback);
+                    this[kGetApi]('Metadata', (error, api) => {
+                        if (error) {
+                            retryCallback(error, undefined);
+                            return;
+                        }
+                        api(connection, options.topics, autocreateTopics, true, retryCallback);
+                    });
                 });
             }, (error, metadata) => {
                 if (error) {
@@ -135,7 +182,7 @@ export class Base extends EventEmitter {
                     brokers.set(broker.nodeId, { host, port });
                 }
                 for (const { name, topicId: id, partitions: rawPartitions, isInternal } of metadata.topics) {
-                    /* c8 ignore next 3 */
+                    /* c8 ignore next 3 - Sometimes internal topics might be returned by Kafka */
                     if (isInternal) {
                         continue;
                     }
@@ -228,6 +275,36 @@ export class Base extends EventEmitter {
         }
         return callback[kCallbackPromise];
     }
+    [kGetApi](name, callback) {
+        // Make sure we have APIs informations
+        if (!this[kApis].length) {
+            this[kListApis]((error, apis) => {
+                if (error) {
+                    callback(error, undefined);
+                    return;
+                }
+                this[kApis] = apis;
+                this[kGetApi](name, callback);
+            });
+            return;
+        }
+        const api = this[kApis].find(api => api.name === name);
+        if (!api) {
+            callback(new UnsupportedApiError(`Unsupported API ${name}.`), undefined);
+            return;
+        }
+        const { minVersion, maxVersion } = api;
+        // Starting from the highest version, we need to find the first one that is supported
+        for (let i = maxVersion; i >= minVersion; i--) {
+            const apiName = (name.slice(0, 1).toLowerCase() + name.slice(1) + 'V' + i);
+            const candidate = apis[apiName];
+            if (candidate) {
+                callback(null, candidate.api);
+                return;
+            }
+        }
+        callback(new UnsupportedApiError(`No usable implementation found for API ${name}.`, { minVersion, maxVersion }), undefined);
+    }
     [kValidateOptions](target, validator, targetName, throwOnErrors = true) {
         if (!this[kOptions].strict) {
             return null;
@@ -242,12 +319,15 @@ export class Base extends EventEmitter {
         }
         return null;
     }
-    // This is a private API used to debug during development
-    /* c8 ignore next 3 */
+    /* c8 ignore next 3 -- This is a private API used to debug during development */
     [kInspect](...args) {
         debugDump(`client:${this[kInstance]}`, ...args);
     }
     [kFormatValidationErrors](validator, targetName) {
         return ajv.errorsText(validator.errors, { dataVar: '$dataVar$' }).replaceAll('$dataVar$', targetName) + '.';
     }
+    [kAfterCreate](type) {
+        this[kClientType] = type;
+        notifyCreation(type, this);
+    }
 }

package/dist/clients/base/options.d.ts

@@ -1,4 +1,6 @@
 import { type BaseOptions } from './types.ts';
+export declare const clientSoftwareName = "platformatic-kafka";
+export declare const clientSoftwareVersion: any;
 export declare const idProperty: {
     type: string;
     pattern: string;

package/dist/clients/base/options.js

@@ -1,4 +1,9 @@
+import { readFileSync } from 'node:fs';
 import { ajv } from "../../utils.js";
+const packageJson = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf-8'));
+// Note: clientSoftwareName can only contain alphanumeric characters, hyphens and dots
+export const clientSoftwareName = 'platformatic-kafka';
+export const clientSoftwareVersion = packageJson.version;
 export const idProperty = { type: 'string', pattern: '^\\S+$' };
 export const topicWithPartitionAndOffsetProperties = {
     topic: idProperty,

package/dist/clients/callbacks.d.ts

@@ -1,5 +1,6 @@
 import { type Callback } from '../apis/definitions.ts';
 export declare const kCallbackPromise: unique symbol;
+export declare const kNoopCallbackReturnValue: unique symbol;
 export declare const noopCallback: CallbackWithPromise<any>;
 export type CallbackWithPromise<ReturnType> = Callback<ReturnType> & {
     [kCallbackPromise]?: Promise<ReturnType>;

package/dist/clients/callbacks.js

@@ -1,8 +1,9 @@
 import { MultipleErrors } from "../errors.js";
 export const kCallbackPromise = Symbol('plt.kafka.callbackPromise');
-/* c8 ignore next 3 */
+// This is only meaningful for testing
+export const kNoopCallbackReturnValue = Symbol('plt.kafka.noopCallbackReturnValue');
 export const noopCallback = () => {
-    return Promise.resolve();
+    return Promise.resolve(kNoopCallbackReturnValue);
 };
 export function createPromisifiedCallback() {
     const { promise, resolve, reject } = Promise.withResolvers();

package/dist/clients/consumer/consumer.d.ts

@@ -1,4 +1,4 @@
-import { type FetchResponse } from '../../apis/consumer/fetch.ts';
+import { type FetchResponse } from '../../apis/consumer/fetch-v17.ts';
 import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { Base, kFetchConnections } from '../base/base.ts';
 import { type CallbackWithPromise } from '../callbacks.ts';