@platformatic/kafka 1.34.0 → 1.35.0

package/dist/clients/base/base.d.ts

@@ -66,6 +66,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     get closed(): boolean;
     get type(): ClientType;
     get context(): unknown;
+    get currentMetadata(): ClusterMetadata | undefined;
     get connections(): ConnectionPool;
     emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
     close(callback: CallbackWithPromise<void>): void;

package/dist/clients/base/base.js

@@ -90,6 +90,9 @@ export class Base extends TypedEventEmitter {
     get context() {
         return this[kContext];
     }
+    get currentMetadata() {
+        return this.#metadata;
+    }
     get connections() {
         return this[kConnections];
     }

package/dist/clients/consumer/consumer.js

@@ -172,6 +172,7 @@ export class Consumer extends Base {
         }
         options.autocommit ??= this[kOptions].autocommit ?? true;
         options.maxBytes ??= this[kOptions].maxBytes;
+        options.maxBytesPerPartition ??= this[kOptions].maxBytesPerPartition ?? options.maxBytes;
         options.highWaterMark ??= this[kOptions].highWaterMark;
         options.registry ??= this[kOptions].registry;
         options.beforeDeserialization ??= this[kOptions].beforeDeserialization;
@@ -618,7 +619,7 @@ export class Consumer extends Base {
             });
         });
     }
-    #listCommittedOffsets(options, callback) {
+    #listCommittedOffsets(options, callback, staleMemberEpochRetries = 0) {
         const topics = [];
         for (const { topic: name, partitions } of options.topics) {
             topics.push({ name, partitionIndexes: partitions });
@@ -640,6 +641,18 @@ export class Consumer extends Base {
             });
         }, (error, response) => {
             if (error) {
+                if (this.#useConsumerGroupProtocol &&
+                    staleMemberEpochRetries < this[kOptions].retries &&
+                    error.findBy?.('apiId', 'STALE_MEMBER_EPOCH')) {
+                    this.#consumerGroupHeartbeat(this[kOptions], heartbeatError => {
+                        if (heartbeatError) {
+                            callback(this.#handleMetadataError(heartbeatError));
+                            return;
+                        }
+                        this.#listCommittedOffsets(options, callback, staleMemberEpochRetries + 1);
+                    });
+                    return;
+                }
                 callback(this.#handleMetadataError(error));
                 return;
             }

package/dist/clients/consumer/messages-stream.js

@@ -306,7 +306,16 @@ export class MessagesStream extends Readable {
         return super[Symbol.asyncIterator]();
     }
     _construct(callback) {
-        this.#refreshOffsets(callback);
+        this.#refreshOffsetsInflight = true;
+        this.#refreshOffsets(error => {
+            const wasPending = this.#refreshOffsetsPending;
+            this.#refreshOffsetsInflight = false;
+            this.#refreshOffsetsPending = false;
+            if (!error && wasPending) {
+                this.#scheduleRefreshOffsetsAndFetch();
+            }
+            callback(error ?? undefined);
+        });
     }
     _destroy(error, callback) {
         if (this.#autocommitInterval) {
@@ -389,7 +398,7 @@ export class MessagesStream extends Readable {
                             {
                                 partition,
                                 fetchOffset,
-                                partitionMaxBytes: this.#options.maxBytes,
+                                partitionMaxBytes: this.#options.maxBytesPerPartition,
                                 currentLeaderEpoch: leaderEpoch,
                                 lastFetchedEpoch: leaderEpoch
                             }
@@ -650,7 +659,10 @@ export class MessagesStream extends Readable {
                 topics.push(assignment);
             }
             if (!topics.length) {
-                this.#assignOffsets(offsets, new Map(), callback);
+                // No assignment yet: do not seed fallback offsets for the whole topic.
+                // consumer:group:join will refresh once partitions are assigned.
+                this.emit('offsets');
+                callback(null);
                 return;
             }
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
@@ -677,6 +689,7 @@ export class MessagesStream extends Readable {
         this.#refreshOffsetsInflight = true;
         this.#refreshOffsets(error => {
             const shouldDestroyOnError = this.#refreshOffsetsDestroyOnError;
+            const wasPending = this.#refreshOffsetsPending;
             this.#refreshOffsetsInflight = false;
             this.#refreshOffsetsDestroyOnError = false;
             this.#refreshOffsetsPending = false;
@@ -688,8 +701,7 @@ export class MessagesStream extends Readable {
                 return;
             }
             // A new one was scheduled while the previous one was inflight, we need to run it immediately
-            /* c8 ignore next 4 - Hard to test */
-            if (this.#refreshOffsetsPending) {
+            if (wasPending) {
                 this.#scheduleRefreshOffsetsAndFetch(shouldDestroyOnError);
                 return;
             }

package/dist/clients/consumer/options.d.ts

@@ -97,6 +97,10 @@ export declare const consumeOptionsProperties: {
         type: string;
         minimum: number;
     };
+    maxBytesPerPartition: {
+        type: string;
+        minimum: number;
+    };
     maxWaitTime: {
         type: string;
         minimum: number;
@@ -217,6 +221,10 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;
@@ -379,6 +387,10 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;
@@ -502,6 +514,10 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        maxBytesPerPartition: {
+            type: string;
+            minimum: number;
+        };
         maxWaitTime: {
             type: string;
             minimum: number;

package/dist/clients/consumer/options.js

@@ -58,6 +58,7 @@ export const consumeOptionsProperties = {
     autocommit: { oneOf: [{ type: 'boolean' }, { type: 'number', minimum: 100 }] },
     minBytes: { type: 'number', minimum: 0 },
     maxBytes: { type: 'number', minimum: 0 },
+    maxBytesPerPartition: { type: 'number', minimum: 0 },
     maxWaitTime: { type: 'number', minimum: 0 },
     isolationLevel: { type: 'number', enum: allowedFetchIsolationLevels },
     deserializers: serdeProperties,

package/dist/clients/consumer/types.d.ts

@@ -65,6 +65,7 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
     autocommit?: boolean | number;
     minBytes?: number;
     maxBytes?: number;
+    maxBytesPerPartition?: number;
     maxWaitTime?: number;
     isolationLevel?: number;
     deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;

package/dist/clients/producer/producer.js

@@ -838,7 +838,7 @@ export class Producer extends Base {
         let partition = 0;
         if (typeof message.partition !== 'number') {
             if (partitioner) {
-                partition = partitioner(message, key);
+                partition = partitioner(message, key, { metadata: this.currentMetadata });
             }
             else if (key) {
                 partition = defaultPartitioner(message, key);

package/dist/clients/producer/types.d.ts

@@ -1,7 +1,7 @@
 import { type CompressionAlgorithmValue } from '../../protocol/compression.ts';
 import { type MessageToProduce } from '../../protocol/records.ts';
 import { type SchemaRegistry } from '../../registries/abstract.ts';
-import { type BaseOptions, type TopicWithPartitionAndOffset } from '../base/types.ts';
+import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from '../base/types.ts';
 import { type BeforeSerializationHook, type Serializers } from '../serde.ts';
 export interface ProducerInfo {
     producerId: bigint;
@@ -12,7 +12,10 @@ export interface ProduceResult {
     offsets?: TopicWithPartitionAndOffset[];
     unwritableNodes?: number[];
 }
-export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key?: Buffer | undefined) => number;
+export interface PartitionerContext {
+    metadata: ClusterMetadata | undefined;
+}
+export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key: Buffer | undefined, context: PartitionerContext) => number;
 export interface ProducerStreamReport {
     batchId: number;
     count: number;

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -66,6 +66,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   get closed(): boolean;
   get type(): ClientType;
   get context(): unknown;
+  get currentMetadata(): ClusterMetadata | undefined;
   get connections(): ConnectionPool;
   emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
   close(callback: CallbackWithPromise<void>): void;

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -97,6 +97,10 @@ export declare const consumeOptionsProperties: {
     type: string;
     minimum: number;
   };
+  maxBytesPerPartition: {
+    type: string;
+    minimum: number;
+  };
   maxWaitTime: {
     type: string;
     minimum: number;
@@ -217,6 +221,10 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;
@@ -379,6 +387,10 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;
@@ -502,6 +514,10 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    maxBytesPerPartition: {
+      type: string;
+      minimum: number;
+    };
     maxWaitTime: {
       type: string;
       minimum: number;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -65,6 +65,7 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
   autocommit?: boolean | number;
   minBytes?: number;
   maxBytes?: number;
+  maxBytesPerPartition?: number;
   maxWaitTime?: number;
   isolationLevel?: number;
   deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;

package/dist/typescript-4/dist/clients/producer/types.d.ts

@@ -1,7 +1,7 @@
 import { type CompressionAlgorithmValue } from "../../protocol/compression";
 import { type MessageToProduce } from "../../protocol/records";
 import { type SchemaRegistry } from "../../registries/abstract";
-import { type BaseOptions, type TopicWithPartitionAndOffset } from "../base/types";
+import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
 import { type BeforeSerializationHook, type Serializers } from "../serde";
 export interface ProducerInfo {
   producerId: bigint;
@@ -12,7 +12,10 @@ export interface ProduceResult {
   offsets?: TopicWithPartitionAndOffset[];
   unwritableNodes?: number[];
 }
-export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key?: Buffer | undefined) => number;
+export interface PartitionerContext {
+  metadata: ClusterMetadata | undefined;
+}
+export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, key: Buffer | undefined, context: PartitionerContext) => number;
 export interface ProducerStreamReport {
   batchId: number;
   count: number;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.34.0";
+export const version = "1.35.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.34.0",
+  "version": "1.35.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -24,6 +24,24 @@
   "type": "module",
   "exports": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "rm -rf dist && tsc -p tsconfig.base.json",
+    "postbuild": "./scripts/node scripts/postbuild.ts",
+    "lint": "eslint --cache",
+    "typecheck": "tsc -p . --noEmit",
+    "format": "prettier -w benchmarks playground src test",
+    "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:memory": "NODE_OPTIONS='--expose-gc' ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/**/*.memory-test.ts'",
+    "test:docker:up": "docker-compose up -d --wait",
+    "test:docker:down": "docker-compose down",
+    "ci": "npm run build && npm run lint && npm run test:ci",
+    "prepublishOnly": "npm run build && npm run lint",
+    "postpublish": "git push origin && git push origin -f --tags",
+    "generate:apis": "node scripts/generate-apis.ts",
+    "generate:errors": "node scripts/generate-errors.ts",
+    "create:api": "node scripts/create-api.ts"
+  },
   "dependencies": {
     "@platformatic/dynamic-buffer": "^0.3.1",
     "@platformatic/wasm-utils": "^0.2.1",
@@ -84,21 +102,5 @@
         "dist/typescript-4/*"
       ]
     }
-  },
-  "scripts": {
-    "build": "rm -rf dist && tsc -p tsconfig.base.json",
-    "postbuild": "./scripts/node scripts/postbuild.ts",
-    "lint": "eslint --cache",
-    "typecheck": "tsc -p . --noEmit",
-    "format": "prettier -w benchmarks playground src test",
-    "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
-    "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
-    "test:memory": "NODE_OPTIONS='--expose-gc' ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/**/*.memory-test.ts'",
-    "test:docker:up": "docker-compose up -d --wait",
-    "test:docker:down": "docker-compose down",
-    "ci": "npm run build && npm run lint && npm run test:ci",
-    "generate:apis": "node scripts/generate-apis.ts",
-    "generate:errors": "node scripts/generate-errors.ts",
-    "create:api": "node scripts/create-api.ts"
   }
 }