@platformatic/kafka 1.9.0 → 1.10.0

package/dist/apis/consumer/fetch-v15.d.ts

@@ -29,7 +29,7 @@ export interface FetchResponsePartition {
     logStartOffset: bigint;
     abortedTransactions: FetchResponsePartitionAbortedTransaction[];
     preferredReadReplica: number;
-    records?: RecordsBatch;
+    records?: RecordsBatch[];
 }
 export interface FetchResponseTopic {
     topicId: string;

package/dist/apis/consumer/fetch-v15.js

@@ -105,7 +105,11 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     }
                     if (recordsSize > 1) {
                         recordsSize--;
-                        partition.records = readRecordsBatch(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
+                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
+                        partition.records = [];
+                        do {
+                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                        } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/consumer/fetch-v16.d.ts

@@ -29,7 +29,7 @@ export interface FetchResponsePartition {
     logStartOffset: bigint;
     abortedTransactions: FetchResponsePartitionAbortedTransaction[];
     preferredReadReplica: number;
-    records?: RecordsBatch;
+    records?: RecordsBatch[];
 }
 export interface FetchResponseTopic {
     topicId: string;

package/dist/apis/consumer/fetch-v16.js

@@ -105,7 +105,11 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     }
                     if (recordsSize > 1) {
                         recordsSize--;
-                        partition.records = readRecordsBatch(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
+                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
+                        partition.records = [];
+                        do {
+                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                        } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/consumer/fetch-v17.d.ts

@@ -29,7 +29,7 @@ export interface FetchResponsePartition {
     logStartOffset: bigint;
     abortedTransactions: FetchResponsePartitionAbortedTransaction[];
     preferredReadReplica: number;
-    records?: RecordsBatch;
+    records?: RecordsBatch[];
 }
 export interface FetchResponseTopic {
     topicId: string;

package/dist/apis/consumer/fetch-v17.js

@@ -105,7 +105,11 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     }
                     if (recordsSize > 1) {
                         recordsSize--;
-                        partition.records = readRecordsBatch(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
+                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
+                        partition.records = [];
+                        do {
+                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                        } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/clients/consumer/messages-stream.d.ts

@@ -9,7 +9,7 @@ export declare function defaultCorruptedMessageHandler(): boolean;
 export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends Readable {
     #private;
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
-    close(callback?: CallbackWithPromise<void>): void;
+    close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
     isActive(): boolean;
     isConnected(): boolean;

package/dist/clients/consumer/messages-stream.js

@@ -291,70 +291,74 @@ export class MessagesStream extends Readable {
         // Parse results
         for (const topicResponse of response.responses) {
             const topic = topicIds.get(topicResponse.topicId);
-            for (const { records, partitionIndex: partition } of topicResponse.partitions) {
-                if (!records) {
+            for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                if (!recordsBatches) {
                     continue;
                 }
-                const firstTimestamp = records.firstTimestamp;
-                const firstOffset = records.firstOffset;
-                const leaderEpoch = metadata.topics.get(topic).partitions[partition].leaderEpoch;
-                for (const record of records.records) {
-                    const offset = records.firstOffset + BigInt(record.offsetDelta);
-                    if (offset < requestedOffsets.get(`${topic}:${partition}`)) {
-                        // Thi is a duplicate message, ignore it
-                        continue;
-                    }
-                    diagnosticContext = createDiagnosticContext({
-                        client: this.#consumer,
-                        stream: this,
-                        operation: 'receive',
-                        raw: record
-                    });
-                    consumerReceivesChannel.start.publish(diagnosticContext);
-                    const commit = autocommit ? noopCallback : this.#commit.bind(this, topic, partition, offset, leaderEpoch);
-                    try {
-                        const headers = new Map();
-                        for (const [headerKey, headerValue] of record.headers) {
-                            headers.set(headerKeyDeserializer(headerKey), headerValueDeserializer(headerValue));
+                for (const batch of recordsBatches) {
+                    const firstTimestamp = batch.firstTimestamp;
+                    const firstOffset = batch.firstOffset;
+                    const leaderEpoch = metadata.topics.get(topic).partitions[partition].leaderEpoch;
+                    for (const record of batch.records) {
+                        const offset = batch.firstOffset + BigInt(record.offsetDelta);
+                        if (offset < requestedOffsets.get(`${topic}:${partition}`)) {
+                            // Thi is a duplicate message, ignore it
+                            continue;
                         }
-                        const key = keyDeserializer(record.key, headers);
-                        const value = valueDeserializer(record.value, headers);
-                        this.#metricsConsumedMessages?.inc();
-                        const message = {
-                            key,
-                            value,
-                            headers,
-                            topic,
-                            partition,
-                            timestamp: firstTimestamp + record.timestampDelta,
-                            offset,
-                            commit
-                        };
-                        diagnosticContext.result = message;
-                        consumerReceivesChannel.asyncStart.publish(diagnosticContext);
-                        canPush = this.push(message);
-                        consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
-                    }
-                    catch (error) {
-                        const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit);
-                        if (shouldDestroy) {
-                            diagnosticContext.error = error;
-                            consumerReceivesChannel.error.publish(diagnosticContext);
-                            this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
-                            return;
+                        diagnosticContext = createDiagnosticContext({
+                            client: this.#consumer,
+                            stream: this,
+                            operation: 'receive',
+                            raw: record
+                        });
+                        consumerReceivesChannel.start.publish(diagnosticContext);
+                        const commit = autocommit ? noopCallback : this.#commit.bind(this, topic, partition, offset, leaderEpoch);
+                        try {
+                            const headers = new Map();
+                            for (const [headerKey, headerValue] of record.headers) {
+                                headers.set(headerKeyDeserializer(headerKey), headerValueDeserializer(headerValue));
+                            }
+                            const key = keyDeserializer(record.key, headers);
+                            const value = valueDeserializer(record.value, headers);
+                            this.#metricsConsumedMessages?.inc();
+                            const message = {
+                                key,
+                                value,
+                                headers,
+                                topic,
+                                partition,
+                                timestamp: firstTimestamp + record.timestampDelta,
+                                offset,
+                                commit
+                            };
+                            diagnosticContext.result = message;
+                            consumerReceivesChannel.asyncStart.publish(diagnosticContext);
+                            canPush = this.push(message);
+                            consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
+                        }
+                        catch (error) {
+                            const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit);
+                            if (shouldDestroy) {
+                                diagnosticContext.error = error;
+                                consumerReceivesChannel.error.publish(diagnosticContext);
+                                this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
+                                return;
+                            }
+                        }
+                        finally {
+                            consumerReceivesChannel.end.publish(diagnosticContext);
                         }
                     }
-                    finally {
-                        consumerReceivesChannel.end.publish(diagnosticContext);
+                    if (batch === recordsBatches[recordsBatches.length - 1]) {
+                        // Track the last read offset
+                        const lastOffset = batch.firstOffset + BigInt(batch.lastOffsetDelta);
+                        this.#offsetsToFetch.set(`${topic}:${partition}`, lastOffset + 1n);
+                        // Autocommit if needed
+                        if (autocommit) {
+                            this.#offsetsToCommit.set(`${topic}:${partition}`, { topic, partition, offset: lastOffset, leaderEpoch });
+                        }
                     }
                 }
-                // Track the last read offset
-                const lastOffset = records.firstOffset + BigInt(records.lastOffsetDelta);
-                this.#offsetsToFetch.set(`${topic}:${partition}`, lastOffset + 1n);
-                // Autocommit if needed
-                if (autocommit) {
-                    this.#offsetsToCommit.set(`${topic}:${partition}`, { topic, partition, offset: lastOffset, leaderEpoch });
-                }
             }
         }
         if (this.#autocommitEnabled && !this.#autocommitInterval) {

package/dist/network/connection.js

@@ -269,46 +269,36 @@ export class Connection extends EventEmitter {
                 request.callback(new NetworkError('Connection closed'), undefined);
                 return false;
             }
-            let canWrite = true;
-            const { correlationId, apiKey, apiVersion, payload: payloadFn, hasRequestHeaderTaggedFields } = request;
-            const writer = Writer.create()
-                .appendInt16(apiKey)
-                .appendInt16(apiVersion)
-                .appendInt32(correlationId)
+            const writer = Writer.create();
+            writer
+                .appendInt16(request.apiKey)
+                .appendInt16(request.apiVersion)
+                .appendInt32(request.correlationId)
                 .appendString(this.#clientId, false);
-            if (hasRequestHeaderTaggedFields) {
+            if (request.hasRequestHeaderTaggedFields) {
                 writer.appendTaggedFields();
             }
-            const payload = payloadFn();
-            writer.appendFrom(payload);
-            writer.prependLength();
-            // Write the header
-            this.#socket.cork();
-            if (!payload.context.noResponse) {
-                this.#inflightRequests.set(correlationId, request);
-            }
-            loggers.protocol('Sending request.', { apiKey: protocolAPIsById[apiKey], correlationId, request });
-            for (const buf of writer.buffers) {
-                if (!this.#socket.write(buf)) {
-                    canWrite = false;
-                }
-            }
-            if (!canWrite) {
+            const payload = request.payload();
+            writer.appendFrom(payload).prependLength();
+            const expectResponse = !payload.context.noResponse;
+            if (expectResponse)
+                this.#inflightRequests.set(request.correlationId, request);
+            const canWrite = this.#socket.write(writer.buffer);
+            if (!canWrite)
                 this.#socketMustBeDrained = true;
-            }
-            this.#socket.uncork();
-            if (payload.context.noResponse) {
+            if (!expectResponse)
                 request.callback(null, canWrite);
-            }
-            // debugDump(Date.now() % 100000, 'send', { owner: this.#ownerId, apiKey: protocolAPIsById[apiKey], correlationId })
+            loggers.protocol('Sending request.', {
+                apiKey: protocolAPIsById[request.apiKey],
+                correlationId: request.correlationId,
+                request
+            });
             return canWrite;
         }
-        catch (error) {
-            request.diagnostic.error = error;
+        catch (err) {
+            request.diagnostic.error = err;
             connectionsApiChannel.error.publish(request.diagnostic);
-            connectionsApiChannel.end.publish(request.diagnostic);
-            throw error;
-            /* c8 ignore next 3 - C8 does not detect these as covered */
+            throw err;
         }
         finally {
             connectionsApiChannel.end.publish(request.diagnostic);

package/dist/protocol/records.js

@@ -60,15 +60,16 @@ export function readRecord(reader) {
 }
 export function createRecordsBatch(messages, options = {}) {
     const now = BigInt(Date.now());
-    const timestamps = [];
-    for (let i = 0; i < messages.length; i++) {
-        timestamps.push(messages[i].timestamp ?? now);
-    }
-    messages.sort();
-    const firstTimestamp = timestamps[0];
-    const maxTimestamp = timestamps[timestamps.length - 1];
+    const firstTimestamp = messages[0].timestamp ?? now;
+    let maxTimestamp = firstTimestamp;
     let buffer = new DynamicBuffer();
     for (let i = 0; i < messages.length; i++) {
+        let ts = messages[i].timestamp ?? now;
+        if (typeof ts === 'number')
+            ts = BigInt(ts);
+        messages[i].timestamp = ts;
+        if (ts > maxTimestamp)
+            maxTimestamp = ts;
         const record = createRecord(messages[i], i, firstTimestamp);
         buffer.appendFrom(record.dynamicBuffer);
     }

package/dist/utils.d.ts

@@ -25,7 +25,7 @@ export declare class NumericMap extends Map<string, number> {
 export declare function niceJoin(array: string[], lastSeparator?: string, separator?: string): string;
 export declare function listErrorMessage(type: string[]): string;
 export declare function enumErrorMessage(type: Record<string, unknown>, keysOnly?: boolean): string;
-export declare function groupByProperty<Key, Value>(entries: Value[], property: keyof Value): [Key, Value[]][];
+export declare function groupByProperty<Key extends PropertyKey, Value>(entries: readonly Value[], property: keyof Value): [Key, Value[]][];
 export declare function humanize(label: string, buffer: Buffer | DynamicBuffer): string;
 export declare function setDebugDumpLogger(logger: DebugDumpLogger): void;
 export declare function debugDump(...values: unknown[]): void;

package/dist/utils.js

@@ -121,19 +121,13 @@ export function enumErrorMessage(type, keysOnly = false) {
     return `should be one of ${niceJoin(Object.entries(type).map(([k, v]) => `${v} (${k})`), ' or ')}`;
 }
 export function groupByProperty(entries, property) {
-    const grouped = new Map();
-    const result = [];
-    for (const entry of entries) {
-        const value = entry[property];
-        let values = grouped.get(value);
-        if (!values) {
-            values = [];
-            grouped.set(value, values);
-            result.push([value, values]);
-        }
-        values.push(entry);
+    const buckets = Object.create(null);
+    for (let i = 0, len = entries.length; i < len; ++i) {
+        const e = entries[i];
+        const key = e[property];
+        (buckets[key] ||= []).push(e);
     }
-    return result;
+    return Object.entries(buckets);
 }
 export function humanize(label, buffer) {
     const formatted = buffer

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.9.0";
+export const version = "1.10.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",