@platformatic/kafka 1.34.0 → 2.0.0

package/README.md

@@ -353,8 +353,7 @@ Many of the methods accept the same options as the client's constructors. The co
 
 Node.js LTS versions:
 
-- `20.19.4` or above
-- `22.18.0` or above
+- `22.22.0` or above
 - `24.6.0` or above
 
 ## Testing

package/dist/apis/callbacks.js

@@ -1,5 +1,4 @@
 import { MultipleErrors } from "../errors.js";
-import { promiseWithResolvers } from "../utils.js";
 export const kCallbackPromise = Symbol('plt.kafka.callbackPromise');
 // This is only meaningful for testing
 export const kNoopCallbackReturnValue = Symbol('plt.kafka.noopCallbackReturnValue');
@@ -7,7 +6,7 @@ export const noopCallback = () => {
     return Promise.resolve(kNoopCallbackReturnValue);
 };
 export function createPromisifiedCallback() {
-    const { promise, resolve, reject } = promiseWithResolvers();
+    const { promise, resolve, reject } = Promise.withResolvers();
     function callback(error, payload) {
         if (error) {
             reject(error);

package/dist/clients/admin/admin.js

@@ -641,7 +641,7 @@ export class Admin extends Base {
                                         const topic = r.readString(false);
                                         return [topic, { topic, partitions: reader.readArray(r => r.readInt32(), false, false) }];
                                     }, false, false);
-                                    reader.readBytes(); // Ignore the user data
+                                    // Ignore the user data
                                 }
                                 group.members.set(member.memberId, {
                                     id: member.memberId,

package/dist/clients/base/base.d.ts

@@ -66,6 +66,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     get closed(): boolean;
     get type(): ClientType;
     get context(): unknown;
+    get currentMetadata(): ClusterMetadata | undefined;
     get connections(): ConnectionPool;
     emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
     close(callback: CallbackWithPromise<void>): void;

package/dist/clients/base/base.js

@@ -90,6 +90,9 @@ export class Base extends TypedEventEmitter {
     get context() {
         return this[kContext];
     }
+    get currentMetadata() {
+        return this.#metadata;
+    }
     get connections() {
         return this[kConnections];
     }

package/dist/clients/consumer/consumer.js

@@ -172,6 +172,7 @@ export class Consumer extends Base {
         }
         options.autocommit ??= this[kOptions].autocommit ?? true;
         options.maxBytes ??= this[kOptions].maxBytes;
+        options.maxBytesPerPartition ??= this[kOptions].maxBytesPerPartition ?? options.maxBytes;
         options.highWaterMark ??= this[kOptions].highWaterMark;
         options.registry ??= this[kOptions].registry;
         options.beforeDeserialization ??= this[kOptions].beforeDeserialization;
@@ -618,7 +619,7 @@ export class Consumer extends Base {
             });
         });
     }
-    #listCommittedOffsets(options, callback) {
+    #listCommittedOffsets(options, callback, staleMemberEpochRetries = 0) {
         const topics = [];
         for (const { topic: name, partitions } of options.topics) {
             topics.push({ name, partitionIndexes: partitions });
@@ -640,6 +641,18 @@ export class Consumer extends Base {
             });
         }, (error, response) => {
             if (error) {
+                if (this.#useConsumerGroupProtocol &&
+                    staleMemberEpochRetries < this[kOptions].retries &&
+                    error.findBy?.('apiId', 'STALE_MEMBER_EPOCH')) {
+                    this.#consumerGroupHeartbeat(this[kOptions], heartbeatError => {
+                        if (heartbeatError) {
+                            callback(this.#handleMetadataError(heartbeatError));
+                            return;
+                        }
+                        this.#listCommittedOffsets(options, callback, staleMemberEpochRetries + 1);
+                    });
+                    return;
+                }
                 callback(this.#handleMetadataError(error));
                 return;
             }
@@ -1229,12 +1242,16 @@ export class Consumer extends Base {
       https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json
     */
     #encodeProtocolAssignment(assignments) {
-        return Writer.create()
+        const userData = this[kOptions].assignmentUserData;
+        const writer = Writer.create()
             .appendInt16(0) // Version information
             .appendArray(assignments, (w, { topic, partitions }) => {
             w.appendString(topic, false).appendArray(partitions, (w, a) => w.appendInt32(a), false, false);
-        }, false, false)
-            .appendInt32(0).buffer; // No user data
+        }, false, false);
+        if (userData) {
+            writer.append(userData);
+        }
+        return writer.buffer;
     }
     #decodeProtocolAssignment(buffer) {
         const reader = Reader.from(buffer);

package/dist/clients/consumer/messages-stream.d.ts

@@ -18,7 +18,6 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     get context(): unknown;
     get offsetsToCommit(): Map<string, CommitOptionsPartition>;
     get offsetsCommitted(): Map<string, bigint>;
-    get committedOffsets(): Map<string, bigint>;
     get connections(): ConnectionPool;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;

package/dist/clients/consumer/messages-stream.js

@@ -187,11 +187,6 @@ export class MessagesStream extends Readable {
     get offsetsCommitted() {
         return this.#offsetsCommitted;
     }
-    // TODO: This is deprecated alias, remove in future major version
-    /* c8 ignore next 3 - Simple getter */
-    get committedOffsets() {
-        return this.#offsetsCommitted;
-    }
     /* c8 ignore next 3 - Simple getter */
     get connections() {
         return this[kConnections];
@@ -389,7 +384,7 @@ export class MessagesStream extends Readable {
                             {
                                 partition,
                                 fetchOffset,
-                                partitionMaxBytes: this.#options.maxBytes,
+                                partitionMaxBytes: this.#options.maxBytesPerPartition,
                                 currentLeaderEpoch: leaderEpoch,
                                 lastFetchedEpoch: leaderEpoch
                             }

package/dist/clients/consumer/options.d.ts

@@ -97,6 +97,10 @@ export declare const consumeOptionsProperties: {
         type: string;
         minimum: number;
     };
+    maxBytesPerPartition: {
+        type: string;
+        minimum: number;
+    };
     maxWaitTime: {
         type: string;
         minimum: number;
@@ -217,6 +221,10 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;
@@ -379,6 +387,10 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;
@@ -502,6 +514,10 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;

package/dist/clients/consumer/options.js

@@ -58,6 +58,7 @@ export const consumeOptionsProperties = {
     autocommit: { oneOf: [{ type: 'boolean' }, { type: 'number', minimum: 100 }] },
     minBytes: { type: 'number', minimum: 0 },
     maxBytes: { type: 'number', minimum: 0 },
+    maxBytesPerPartition: { type: 'number', minimum: 0 },
     maxWaitTime: { type: 'number', minimum: 0 },
     isolationLevel: { type: 'number', enum: allowedFetchIsolationLevels },
     deserializers: serdeProperties,
@@ -245,7 +246,7 @@ export const defaultConsumerOptions = {
     heartbeatInterval: 3000,
     protocols: [{ name: 'roundrobin', version: 1 }],
     minBytes: 1,
-    maxBytes: 1_048_576 * 10, // 10 MB
+    maxBytes: 1_048_576 * 50, // 50 MB
     maxWaitTime: 5_000,
     isolationLevel: FetchIsolationLevels.READ_COMMITTED,
     highWaterMark: 1024

package/dist/clients/consumer/types.d.ts

@@ -54,17 +54,20 @@ export interface GroupOptions {
     heartbeatInterval?: number;
     protocols?: GroupProtocolSubscription[];
     partitionAssigner?: GroupPartitionsAssigner;
+    assignmentUserData?: Buffer;
 }
 export interface ConsumerGroupOptions {
     groupProtocol: typeof GroupProtocols.CONSUMER;
     groupInstanceId?: string;
     groupRemoteAssignor?: string;
     rebalanceTimeout?: number;
+    assignmentUserData?: Buffer;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
     autocommit?: boolean | number;
     minBytes?: number;
     maxBytes?: number;
+    maxBytesPerPartition?: number;
     maxWaitTime?: number;
     isolationLevel?: number;
     deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;

package/dist/clients/producer/producer.js

@@ -838,7 +838,7 @@ export class Producer extends Base {
         let partition = 0;
         if (typeof message.partition !== 'number') {
             if (partitioner) {
-                partition = partitioner(message, key);
+                partition = partitioner(message, key, { metadata: this.currentMetadata });
             }
             else if (key) {
                 partition = defaultPartitioner(message, key);

package/dist/clients/producer/types.d.ts

@@ -1,7 +1,7 @@
 import { type CompressionAlgorithmValue } from '../../protocol/compression.ts';
 import { type MessageToProduce } from '../../protocol/records.ts';
 import { type SchemaRegistry } from '../../registries/abstract.ts';
-import { type BaseOptions, type TopicWithPartitionAndOffset } from '../base/types.ts';
+import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from '../base/types.ts';
 import { type BeforeSerializationHook, type Serializers } from '../serde.ts';
 export interface ProducerInfo {
     producerId: bigint;
@@ -12,7 +12,10 @@ export interface ProduceResult {
     offsets?: TopicWithPartitionAndOffset[];
     unwritableNodes?: number[];
 }
-export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key?: Buffer | undefined) => number;
+export interface PartitionerContext {
+    metadata: ClusterMetadata | undefined;
+}
+export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key: Buffer | undefined, context: PartitionerContext) => number;
 export interface ProducerStreamReport {
     batchId: number;
     count: number;

package/dist/protocol/compression.js

@@ -15,9 +15,31 @@ function ensureBuffer(data) {
     return DynamicBuffer.isDynamicBuffer(data) ? data.buffer : data;
 }
 const snappyCompressSync = snappyCompress;
-const snappyDecompressSync = snappyDecompress;
 const lz4CompressFrameSync = lz4Compress;
 const lz4DecompressFrameSync = lz4Decompress;
+const xerialSnappyHeader = Buffer.from([0x82, 0x53, 0x4e, 0x41, 0x50, 0x50, 0x59, 0x00]);
+function snappyDecompressSync(data) {
+    if (!data.subarray(0, xerialSnappyHeader.length).equals(xerialSnappyHeader)) {
+        return snappyDecompress(data);
+    }
+    const decompressed = new DynamicBuffer();
+    let offset = 16;
+    while (offset < data.length) {
+        /* c8 ignore next 3 - Hard to test */
+        if (offset + 4 > data.length) {
+            throw new Error('Invalid xerial snappy chunk length');
+        }
+        const chunkLength = data.readUInt32BE(offset);
+        offset += 4;
+        /* c8 ignore next 3 - Hard to test */
+        if (offset + chunkLength > data.length) {
+            throw new Error('Invalid xerial snappy chunk data');
+        }
+        decompressed.append(snappyDecompress(data.subarray(offset, offset + chunkLength)));
+        offset += chunkLength;
+    }
+    return decompressed.buffer;
+}
 export const compressionsAlgorithms = {
     /* c8 ignore next 8 - 'none' is actually never used but this is to please Typescript */
     none: {

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -66,6 +66,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   get closed(): boolean;
   get type(): ClientType;
   get context(): unknown;
+  get currentMetadata(): ClusterMetadata | undefined;
   get connections(): ConnectionPool;
   emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
   close(callback: CallbackWithPromise<void>): void;

package/dist/typescript-4/dist/clients/consumer/messages-stream.d.ts

@@ -18,7 +18,6 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
   get context(): unknown;
   get offsetsToCommit(): Map<string, CommitOptionsPartition>;
   get offsetsCommitted(): Map<string, bigint>;
-  get committedOffsets(): Map<string, bigint>;
   get connections(): ConnectionPool;
   close(callback: CallbackWithPromise<void>): void;
   close(): Promise<void>;

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -97,6 +97,10 @@ export declare const consumeOptionsProperties: {
     type: string;
     minimum: number;
   };
+  maxBytesPerPartition: {
+    type: string;
+    minimum: number;
+  };
   maxWaitTime: {
     type: string;
     minimum: number;
@@ -217,6 +221,10 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;
@@ -379,6 +387,10 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;
@@ -502,6 +514,10 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -54,17 +54,20 @@ export interface GroupOptions {
   heartbeatInterval?: number;
   protocols?: GroupProtocolSubscription[];
   partitionAssigner?: GroupPartitionsAssigner;
+  assignmentUserData?: Buffer;
 }
 export interface ConsumerGroupOptions {
   groupProtocol: typeof GroupProtocols.CONSUMER;
   groupInstanceId?: string;
   groupRemoteAssignor?: string;
   rebalanceTimeout?: number;
+  assignmentUserData?: Buffer;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
   autocommit?: boolean | number;
   minBytes?: number;
   maxBytes?: number;
+  maxBytesPerPartition?: number;
   maxWaitTime?: number;
   isolationLevel?: number;
   deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;

package/dist/typescript-4/dist/clients/producer/types.d.ts

@@ -1,7 +1,7 @@
 import { type CompressionAlgorithmValue } from "../../protocol/compression";
 import { type MessageToProduce } from "../../protocol/records";
 import { type SchemaRegistry } from "../../registries/abstract";
-import { type BaseOptions, type TopicWithPartitionAndOffset } from "../base/types";
+import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
 import { type BeforeSerializationHook, type Serializers } from "../serde";
 export interface ProducerInfo {
   producerId: bigint;
@@ -12,7 +12,10 @@ export interface ProduceResult {
   offsets?: TopicWithPartitionAndOffset[];
   unwritableNodes?: number[];
 }
-export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key?: Buffer | undefined) => number;
+export interface PartitionerContext {
+  metadata: ClusterMetadata | undefined;
+}
+export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key: Buffer | undefined, context: PartitionerContext) => number;
 export interface ProducerStreamReport {
   batchId: number;
   count: number;

package/dist/typescript-4/dist/utils.d.ts

@@ -15,12 +15,6 @@ export interface DataValidationContext {
 }
 export type DebugDumpLogger = (...args: any[]) => void;
 export { setTimeout as sleep } from 'node:timers/promises';
-interface PromiseWithResolvers<T> {
-  promise: Promise<T>;
-  resolve: (value: T | PromiseLike<T>) => void;
-  reject: (reason?: any) => void;
-}
-export declare const promiseWithResolvers: <T>() => PromiseWithResolvers<T>;
 export declare const ajv: Ajv2020;
 export declare const loggers: Record<string, debug.Debugger>;
 export declare class NumericMap extends Map<string, number> {

package/dist/utils.d.ts

@@ -15,12 +15,6 @@ export interface DataValidationContext {
 }
 export type DebugDumpLogger = (...args: any[]) => void;
 export { setTimeout as sleep } from 'node:timers/promises';
-interface PromiseWithResolvers<T> {
-    promise: Promise<T>;
-    resolve: (value: T | PromiseLike<T>) => void;
-    reject: (reason?: any) => void;
-}
-export declare const promiseWithResolvers: <T>() => PromiseWithResolvers<T>;
 export declare const ajv: Ajv2020;
 export declare const loggers: Record<string, debug.Debugger>;
 export declare class NumericMap extends Map<string, number> {

package/dist/utils.js

@@ -2,20 +2,6 @@ import { Ajv2020 } from 'ajv/dist/2020.js';
 import debug from 'debug';
 import { inspect } from 'node:util';
 export { setTimeout as sleep } from 'node:timers/promises';
-function promiseWithResolversPolyfill() {
-    let resolve;
-    let reject;
-    const promise = new Promise((_resolve, _reject) => {
-        resolve = _resolve;
-        reject = _reject;
-    });
-    // @ts-expect-error - resolve and reject are assigned in the promise constructor
-    return { promise, resolve, reject };
-}
-export const promiseWithResolvers = Promise.withResolvers
-    ? Promise.withResolvers.bind(Promise)
-    : promiseWithResolversPolyfill;
-/* c8 ignore end */
 export const ajv = new Ajv2020({ allErrors: true, coerceTypes: false, strict: true });
 export const loggers = {
     protocol: debug('plt:kafka:protocol'),

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.34.0";
+export const version = "2.0.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.34.0",
+  "version": "2.0.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -59,7 +59,6 @@
     "cronometro": "^5.3.0",
     "eslint": "^9.35.0",
     "fast-jwt": "^6.0.2",
-    "glob": "^13.0.0",
     "hwp": "^0.4.1",
     "json5": "^2.2.3",
     "kafkajs": "^2.2.4",
@@ -76,7 +75,7 @@
     "typescript": "^5.9.2"
   },
   "engines": {
-    "node": ">= 20.19.4 || >= 22.18.0 || >= 24.6.0"
+    "node": ">= 22.22.0 || >= 24.6.0"
   },
   "typesVersions": {
     "<5.0": {