@platformatic/kafka 1.18.0 → 1.20.0

package/dist/clients/admin/admin.js

@@ -364,18 +364,22 @@ export class Admin extends Base {
                             };
                             for (const member of raw.members) {
                                 const reader = Reader.from(member.memberMetadata);
-                                const memberMetadata = {
-                                    version: reader.readInt16(),
-                                    topics: reader.readArray(r => r.readString(false), false, false),
-                                    metadata: reader.readBytes(false)
-                                };
-                                reader.reset(member.memberAssignment);
-                                reader.skip(2); // Ignore Version information
-                                const memberAssignments = reader.readMap(r => {
-                                    const topic = r.readString(false);
-                                    return [topic, { topic, partitions: reader.readArray(r => r.readInt32(), false, false) }];
-                                }, false, false);
-                                reader.readBytes(); // Ignore the user data
+                                let memberMetadata;
+                                let memberAssignments;
+                                if (reader.remaining > 0) {
+                                    memberMetadata = {
+                                        version: reader.readInt16(),
+                                        topics: reader.readArray(r => r.readString(false), false, false),
+                                        metadata: reader.readBytes(false)
+                                    };
+                                    reader.reset(member.memberAssignment);
+                                    reader.skip(2); // Ignore Version information
+                                    memberAssignments = reader.readMap(r => {
+                                        const topic = r.readString(false);
+                                        return [topic, { topic, partitions: reader.readArray(r => r.readInt32(), false, false) }];
+                                    }, false, false);
+                                    reader.readBytes(); // Ignore the user data
+                                }
                                 group.members.set(member.memberId, {
                                     id: member.memberId,
                                     groupInstanceId: member.groupInstanceId,

package/dist/clients/admin/types.d.ts

@@ -1,7 +1,7 @@
 import { type AlterClientQuotasRequestEntry } from '../../apis/admin/alter-client-quotas-v1.ts';
-import { type DescribeClientQuotasRequestComponent } from '../../apis/admin/describe-client-quotas-v0.ts';
-import { type DescribeLogDirsResponse, type DescribeLogDirsResponseResult, type DescribeLogDirsRequestTopic } from '../../apis/admin/describe-log-dirs-v4.ts';
 import { type CreateTopicsRequestTopicConfig } from '../../apis/admin/create-topics-v7.ts';
+import { type DescribeClientQuotasRequestComponent } from '../../apis/admin/describe-client-quotas-v0.ts';
+import { type DescribeLogDirsRequestTopic, type DescribeLogDirsResponse, type DescribeLogDirsResponseResult } from '../../apis/admin/describe-log-dirs-v4.ts';
 import { type ConsumerGroupState } from '../../apis/enumerations.ts';
 import { type NullableString } from '../../protocol/definitions.ts';
 import { type BaseOptions } from '../base/types.ts';
@@ -22,8 +22,8 @@ export interface GroupMember {
     groupInstanceId: NullableString;
     clientId: string;
     clientHost: string;
-    metadata: Omit<ExtendedGroupProtocolSubscription, 'memberId'>;
-    assignments: Map<string, GroupAssignment>;
+    metadata?: Omit<ExtendedGroupProtocolSubscription, 'memberId'>;
+    assignments?: Map<string, GroupAssignment>;
 }
 export interface GroupBase {
     id: string;

package/dist/clients/consumer/consumer.js

@@ -271,7 +271,7 @@ export class Consumer extends Base {
             // Now gather the last committed offsets from each stream
             const committeds = new Map();
             for (const stream of this.#streams) {
-                for (const [topic, offset] of stream.committedOffsets) {
+                for (const [topic, offset] of stream.offsetsCommitted) {
                     committeds.set(topic, offset);
                 }
             }
@@ -448,13 +448,17 @@ export class Consumer extends Base {
         });
     }
     #listOffsets(withTimestamps, options, callback) {
-        this[kMetadata]({ topics: options.topics }, (error, metadata) => {
+        let topics = options.topics;
+        if (!topics || topics.length === 0) {
+            topics = this.topics.current;
+        }
+        this[kMetadata]({ topics }, (error, metadata) => {
             if (error) {
                 callback(error, undefined);
                 return;
             }
             const requests = new Map();
-            for (const name of options.topics) {
+            for (const name of topics) {
                 const topic = metadata.topics.get(name);
                 const toInclude = new Set(options.partitions?.[name] ?? []);
                 const hasPartitionsFilter = toInclude.size > 0;

package/dist/clients/consumer/messages-stream.d.ts

@@ -11,6 +11,9 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     #private;
     [kInstance]: number;
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
+    get offsetsToFetch(): Map<string, bigint>;
+    get offsetsToCommit(): Map<string, CommitOptionsPartition>;
+    get offsetsCommitted(): Map<string, bigint>;
     get committedOffsets(): Map<string, bigint>;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
@@ -20,6 +23,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     pause(): this;
     addListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     addListener(event: 'fetch', listener: () => void): this;
+    addListener(event: 'offsets', listener: () => void): this;
     addListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     addListener(event: 'close', listener: () => void): this;
     addListener(event: 'end', listener: () => void): this;
@@ -29,6 +33,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     addListener(event: 'resume', listener: () => void): this;
     on(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     on(event: 'fetch', listener: () => void): this;
+    on(event: 'offsets', listener: () => void): this;
     on(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     on(event: 'close', listener: () => void): this;
     on(event: 'end', listener: () => void): this;
@@ -38,6 +43,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     on(event: 'resume', listener: () => void): this;
     once(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     once(event: 'fetch', listener: () => void): this;
+    once(event: 'offsets', listener: () => void): this;
     once(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     once(event: 'close', listener: () => void): this;
     once(event: 'end', listener: () => void): this;
@@ -47,6 +53,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     once(event: 'resume', listener: () => void): this;
     prependListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     prependListener(event: 'fetch', listener: () => void): this;
+    prependListener(event: 'offsets', listener: () => void): this;
     prependListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     prependListener(event: 'close', listener: () => void): this;
     prependListener(event: 'end', listener: () => void): this;
@@ -56,6 +63,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     prependListener(event: 'resume', listener: () => void): this;
     prependOnceListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     prependOnceListener(event: 'fetch', listener: () => void): this;
+    prependOnceListener(event: 'offsets', listener: () => void): this;
     prependOnceListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     prependOnceListener(event: 'close', listener: () => void): this;
     prependOnceListener(event: 'end', listener: () => void): this;

package/dist/clients/consumer/messages-stream.js

@@ -98,6 +98,7 @@ export class MessagesStream extends Readable {
         // will have changed so we may have gone from last  with no assignments to
         // having some.
         this.#consumer.on('consumer:group:join', () => {
+            this.#offsetsCommitted.clear();
             this.#refreshOffsets((error) => {
                 /* c8 ignore next 4 - Hard to test */
                 if (error) {
@@ -112,6 +113,16 @@ export class MessagesStream extends Readable {
         }
         notifyCreation('messages-stream', this);
     }
+    get offsetsToFetch() {
+        return this.#offsetsToFetch;
+    }
+    get offsetsToCommit() {
+        return this.#offsetsToCommit;
+    }
+    get offsetsCommitted() {
+        return this.#offsetsCommitted;
+    }
+    // TODO: This is deprecated alias, remove in future major version
     get committedOffsets() {
         return this.#offsetsCommitted;
     }
@@ -525,6 +536,20 @@ export class MessagesStream extends Readable {
                 }
             }
         }
+        // Rebuild the list of offsetsCommitted (which is used for consumer lag) out of the offsets to fetch
+        for (const topic of this.#topics) {
+            const assignment = this.#assignmentsForTopic(topic);
+            // This consumer has no assignment for the topic, continue
+            if (!assignment) {
+                continue;
+            }
+            const partitions = assignment.partitions;
+            for (const partition of partitions) {
+                const committed = this.#offsetsToFetch.get(`${topic}:${partition}`);
+                this.#offsetsCommitted.set(`${topic}:${partition}`, committed - 1n);
+            }
+        }
+        this.emit('offsets');
         callback(null);
     }
     #assignmentsForTopic(topic) {

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.18.0";
+export const version = "1.20.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.18.0",
+  "version": "1.20.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",