kafka-ts 1.0.1 → 1.0.3

package/README.md

@@ -167,7 +167,6 @@ Custom SASL mechanisms can be implemented following the `SASLProvider` interface
 | partitionMaxBytes      | number                                 | false    | 1_048_576                       | Maximum number of bytes to return per partition in the fetch response                |
 | allowTopicAutoCreation | boolean                                | false    | false                           | Allow kafka to auto-create topic when it doesn't exist                               |
 | fromTimestamp          | bigint                                 | false    | -1                              | Start consuming messages from timestamp (-1 = latest offsets, -2 = earliest offsets) |
-| batchSize              | number                                 | false    | null                            | Maximum number of records called `onBatch`                                           |
 | onBatch                | (batch: Message[]) => Promise<unknown> | true     |                                 | Callback executed when a batch of messages is received                               |
 
 ### `kafka.createProducer()`

package/dist/consumer/consumer-group.js

@@ -157,9 +157,13 @@ class ConsumerGroup {
             groupInstanceId,
             memberId: this.memberId,
             generationIdOrMemberEpoch: this.generationId,
-            topics: Object.entries(topicPartitions).map(([topic, partitions]) => ({
+            topics: Object.entries(topicPartitions)
+                .filter(([topic]) => topic in offsetManager.pendingOffsets)
+                .map(([topic, partitions]) => ({
                 name: topic,
-                partitions: [...partitions].map((partitionIndex) => ({
+                partitions: [...partitions]
+                    .filter((partition) => partition in offsetManager.pendingOffsets[topic])
+                    .map((partitionIndex) => ({
                     partitionIndex,
                     committedOffset: offsetManager.pendingOffsets[topic][partitionIndex],
                     committedLeaderEpoch: -1,

package/dist/consumer/consumer.d.ts

@@ -19,7 +19,6 @@ export type ConsumerOptions = {
     allowTopicAutoCreation?: boolean;
     fromBeginning?: boolean;
     fromTimestamp?: bigint;
-    batchSize?: number | null;
     retrier?: Retrier;
     onBatch: (messages: Required<Message>[]) => unknown;
 };

package/dist/consumer/consumer.js

@@ -48,11 +48,10 @@ class Consumer extends events_1.default {
             minBytes: options.minBytes ?? 1,
             maxBytes: options.maxBytes ?? 1_048_576,
             partitionMaxBytes: options.partitionMaxBytes ?? 1_048_576,
-            isolationLevel: options.isolationLevel ?? 0 /* IsolationLevel.READ_UNCOMMITTED */,
+            isolationLevel: options.isolationLevel ?? 1 /* IsolationLevel.READ_COMMITTED */,
             allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
             fromBeginning: options.fromBeginning ?? false,
             fromTimestamp: options.fromTimestamp ?? (options.fromBeginning ? -2n : -1n),
-            batchSize: options.batchSize ?? null,
             retrier: options.retrier ?? retrier_1.defaultRetrier,
         };
         this.metadata = new consumer_metadata_1.ConsumerMetadata({ cluster: this.cluster });
@@ -106,7 +105,7 @@ class Consumer extends events_1.default {
         await this.cluster.disconnect().catch((error) => logger_1.log.debug(`Failed to disconnect: ${error.message}`));
     }
     async startFetchManager() {
-        const { groupId, batchSize } = this.options;
+        const { groupId } = this.options;
         while (!this.stopHook) {
             try {
                 await this.consumerGroup?.join();
@@ -121,9 +120,6 @@ class Consumer extends events_1.default {
                 this.fetchManager = new fetch_manager_1.FetchManager({
                     fetch: this.fetch.bind(this),
                     process: this.process.bind(this),
-                    metadata: this.metadata,
-                    consumerGroup: this.consumerGroup,
-                    batchSize,
                     nodeAssignments,
                 });
                 await this.fetchManager.start();
@@ -165,21 +161,44 @@ class Consumer extends events_1.default {
             this.consumerGroup?.handleLastHeartbeat();
         }
     }
-    async process(messages) {
+    async process(response) {
         const { options } = this;
         const { retrier } = options;
+        this.consumerGroup?.handleLastHeartbeat();
         const topicPartitions = {};
-        for (const { topic, partition } of messages) {
+        const messages = response.responses.flatMap(({ topicId, partitions }) => {
+            const topic = this.metadata.getTopicNameById(topicId);
             topicPartitions[topic] ??= new Set();
-            topicPartitions[topic].add(partition);
+            return partitions.flatMap(({ partitionIndex, records }) => {
+                topicPartitions[topic].add(partitionIndex);
+                return records.flatMap(({ baseTimestamp, baseOffset, records }) => records.flatMap((message) => ({
+                    topic,
+                    partition: partitionIndex,
+                    key: message.key ?? null,
+                    value: message.value ?? null,
+                    headers: Object.fromEntries(message.headers.map(({ key, value }) => [key, value])),
+                    timestamp: baseTimestamp + BigInt(message.timestampDelta),
+                    offset: baseOffset + BigInt(message.offsetDelta),
+                })));
+            });
+        });
+        if (!messages.length) {
+            return;
         }
         await retrier(() => options.onBatch(messages));
-        messages.forEach(({ topic, partition, offset }) => this.offsetManager.resolve(topic, partition, offset + 1n));
+        response.responses.forEach(({ topicId, partitions }) => {
+            partitions.forEach(({ partitionIndex, records }) => {
+                records.forEach(({ baseOffset, lastOffsetDelta }) => {
+                    this.offsetManager.resolve(this.metadata.getTopicNameById(topicId), partitionIndex, baseOffset + BigInt(lastOffsetDelta) + 1n);
+                });
+            });
+        });
         await this.consumerGroup?.offsetCommit(topicPartitions);
         this.offsetManager.flush(topicPartitions);
     }
     fetch(nodeId, assignment) {
         const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;
+        this.consumerGroup?.handleLastHeartbeat();
         return this.cluster.sendRequestToNode(nodeId)(api_1.API.FETCH, {
             maxWaitMs,
             minBytes,
@@ -194,7 +213,7 @@ class Consumer extends events_1.default {
                     currentLeaderEpoch: -1,
                     fetchOffset: this.offsetManager.getCurrentOffset(topic, partition),
                     lastFetchedEpoch: -1,
-                    logStartOffset: 0n,
+                    logStartOffset: -1n,
                     partitionMaxBytes,
                 })),
             })),
@@ -217,8 +236,8 @@ __decorate([
     __metadata("design:returntype", Promise)
 ], Consumer.prototype, "close", null);
 __decorate([
-    trace((messages) => ({ count: messages.length })),
+    trace(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [Array]),
+    __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], Consumer.prototype, "process", null);

package/dist/consumer/fetch-manager.d.ts

@@ -1,14 +1,8 @@
 import { FetchResponse } from '../api/fetch';
 import { Assignment } from '../api/sync-group';
-import { Metadata } from '../metadata';
-import { Batch, Message } from '../types';
-import { ConsumerGroup } from './consumer-group';
 type FetchManagerOptions = {
     fetch: (nodeId: number, assignment: Assignment) => Promise<FetchResponse>;
-    process: (batch: Batch) => Promise<void>;
-    batchSize?: number | null;
-    metadata: Metadata;
-    consumerGroup?: ConsumerGroup;
+    process: (response: FetchResponse) => Promise<void>;
     nodeAssignments: {
         nodeId: number;
         assignment: Assignment;
@@ -16,16 +10,9 @@ type FetchManagerOptions = {
 };
 export declare class FetchManager {
     private options;
-    private queue;
-    private isRunning;
     private fetchers;
-    private processor;
-    private pollCallback;
-    private fetcherCallbacks;
     constructor(options: FetchManagerOptions);
     start(): Promise<void>;
-    stop(): Promise<void>;
-    poll(): Promise<Required<Message>[]>;
-    private onResponse;
+    stop(): Promise<void[]>;
 }
 export {};

package/dist/consumer/fetch-manager.js

@@ -12,82 +12,25 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.FetchManager = void 0;
 const tracer_1 = require("../utils/tracer");
 const fetcher_1 = require("./fetcher");
-const processor_1 = require("./processor");
 const trace = (0, tracer_1.createTracer)('FetchManager');
 class FetchManager {
     options;
-    queue = [];
-    isRunning = false;
     fetchers;
-    processor;
-    pollCallback;
-    fetcherCallbacks = {};
     constructor(options) {
         this.options = options;
         const { fetch, process, nodeAssignments } = this.options;
-        this.fetchers = nodeAssignments.map(({ nodeId, assignment }, index) => new fetcher_1.Fetcher(index, {
-            nodeId,
-            assignment,
-            fetch,
-            onResponse: this.onResponse.bind(this),
-        }));
-        this.processor = new processor_1.Processor({ process, poll: this.poll.bind(this) });
+        this.fetchers = nodeAssignments.map(({ nodeId, assignment }) => new fetcher_1.Fetcher({ nodeId, assignment, fetch, process }));
     }
     async start() {
-        this.queue = [];
-        this.isRunning = true;
         try {
-            await Promise.all([...this.fetchers.map((fetcher) => fetcher.loop()), this.processor.loop()]);
+            await Promise.all(this.fetchers.map((fetcher) => fetcher.loop()));
         }
         finally {
             await this.stop();
         }
     }
     async stop() {
-        this.isRunning = false;
-        const stopPromise = Promise.all([...this.fetchers.map((fetcher) => fetcher.stop()), this.processor.stop()]);
-        this.pollCallback?.();
-        Object.values(this.fetcherCallbacks).forEach((callback) => callback());
-        this.fetcherCallbacks = {};
-        await stopPromise;
-    }
-    async poll() {
-        if (!this.isRunning) {
-            return [];
-        }
-        const { consumerGroup, batchSize } = this.options;
-        consumerGroup?.handleLastHeartbeat();
-        const batch = batchSize ? this.queue.splice(0, batchSize) : this.queue.splice(0);
-        if (!batch.length) {
-            await new Promise((resolve) => (this.pollCallback = resolve));
-            return this.poll();
-        }
-        const [checkpoints, messages] = partition(batch, (entry) => 'kind' in entry && entry.kind === 'checkpoint');
-        checkpoints.forEach(({ fetcherId }) => this.fetcherCallbacks[fetcherId]?.());
-        return messages;
-    }
-    async onResponse(fetcherId, response) {
-        const { metadata, consumerGroup } = this.options;
-        consumerGroup?.handleLastHeartbeat();
-        const messages = response.responses.flatMap(({ topicId, partitions }) => partitions.flatMap(({ partitionIndex, records }) => records.flatMap(({ baseTimestamp, baseOffset, records }) => records.flatMap((message) => ({
-            topic: metadata.getTopicNameById(topicId),
-            partition: partitionIndex,
-            key: message.key ?? null,
-            value: message.value ?? null,
-            headers: Object.fromEntries(message.headers.map(({ key, value }) => [key, value])),
-            timestamp: baseTimestamp + BigInt(message.timestampDelta),
-            offset: baseOffset + BigInt(message.offsetDelta),
-        })))));
-        if (!messages.length) {
-            return;
-        }
-        // wait until all broker batches have been processed or fetch manager is requested to stop
-        await new Promise((resolve) => {
-            this.fetcherCallbacks[fetcherId] = resolve;
-            this.queue.push(...messages, { kind: 'checkpoint', fetcherId });
-            this.pollCallback?.();
-        });
-        consumerGroup?.handleLastHeartbeat();
+        return Promise.all(this.fetchers.map((fetcher) => fetcher.stop()));
     }
 }
 exports.FetchManager = FetchManager;
@@ -97,28 +40,3 @@ __decorate([
     __metadata("design:paramtypes", []),
     __metadata("design:returntype", Promise)
 ], FetchManager.prototype, "start", null);
-__decorate([
-    trace(),
-    __metadata("design:type", Function),
-    __metadata("design:paramtypes", []),
-    __metadata("design:returntype", Promise)
-], FetchManager.prototype, "poll", null);
-__decorate([
-    trace(),
-    __metadata("design:type", Function),
-    __metadata("design:paramtypes", [Number, Object]),
-    __metadata("design:returntype", Promise)
-], FetchManager.prototype, "onResponse", null);
-const partition = (batch, predicate) => {
-    const checkpoints = [];
-    const messages = [];
-    for (const entry of batch) {
-        if (predicate(entry)) {
-            checkpoints.push(entry);
-        }
-        else {
-            messages.push(entry);
-        }
-    }
-    return [checkpoints, messages];
-};

package/dist/consumer/fetcher.d.ts

@@ -7,15 +7,14 @@ type FetcherOptions = {
     nodeId: number;
     assignment: Assignment;
     fetch: (nodeId: number, assignment: Assignment) => Promise<FetchResponse>;
-    onResponse: (fetcherId: number, response: FetchResponse) => Promise<void>;
+    process: (response: FetchResponse) => Promise<void>;
 };
 export declare class Fetcher extends EventEmitter<{
     stopped: [];
 }> {
-    private fetcherId;
     private options;
     private isRunning;
-    constructor(fetcherId: number, options: FetcherOptions);
+    constructor(options: FetcherOptions);
     loop(): Promise<void>;
     private step;
     stop(): Promise<void>;

package/dist/consumer/fetcher.js

@@ -14,12 +14,10 @@ const stream_1 = require("stream");
 const tracer_1 = require("../utils/tracer");
 const trace = (0, tracer_1.createTracer)('Fetcher');
 class Fetcher extends stream_1.EventEmitter {
-    fetcherId;
     options;
     isRunning = false;
-    constructor(fetcherId, options) {
+    constructor(options) {
         super();
-        this.fetcherId = fetcherId;
         this.options = options;
     }
     async loop() {
@@ -35,12 +33,11 @@ class Fetcher extends stream_1.EventEmitter {
         }
     }
     async step() {
-        const { nodeId, assignment, fetch, onResponse } = this.options;
+        const { nodeId, assignment, fetch, process } = this.options;
         const response = await fetch(nodeId, assignment);
-        if (!this.isRunning) {
+        if (!this.isRunning)
             return;
-        }
-        await onResponse(this.fetcherId, response);
+        await process(response);
     }
     async stop() {
         if (!this.isRunning) {

package/dist/metadata.d.ts

@@ -16,11 +16,11 @@ export declare class Metadata {
     getTopicIdByName(name: string): string;
     getTopicNameById(id: string): string;
     fetchMetadataIfNecessary({ topics, allowTopicAutoCreation, }: {
-        topics: string[];
+        topics: string[] | Set<string>;
         allowTopicAutoCreation: boolean;
     }): Promise<void>;
     fetchMetadata({ topics, allowTopicAutoCreation, }: {
-        topics: string[] | null;
+        topics: string[] | Set<string> | null;
         allowTopicAutoCreation: boolean;
     }): Promise<void>;
 }

package/dist/metadata.js

@@ -41,7 +41,7 @@ class Metadata {
         return this.topicNameById[id];
     }
     async fetchMetadataIfNecessary({ topics, allowTopicAutoCreation, }) {
-        const missingTopics = topics.filter((topic) => !this.topicPartitions[topic]);
+        const missingTopics = Array.from(topics).filter((topic) => !this.topicPartitions[topic]);
         if (!missingTopics.length) {
             return;
         }
@@ -63,7 +63,7 @@ class Metadata {
         const { cluster } = this.options;
         const response = await cluster.sendRequest(api_1.API.METADATA, {
             allowTopicAutoCreation,
-            topics: topics?.map((name) => ({ id: null, name })) ?? null,
+            topics: topics ? Array.from(topics).map((name) => ({ id: null, name })) : null,
         });
         this.topicPartitions = {
             ...this.topicPartitions,

package/dist/producer/producer.js

@@ -40,9 +40,13 @@ class Producer {
         await this.ensureConnected();
         const { allowTopicAutoCreation } = this.options;
         const defaultTimestamp = BigInt(Date.now());
-        const topics = Array.from(new Set(messages.map((message) => message.topic)));
+        const topics = new Set(messages.map((message) => message.topic));
         await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
-        const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(messages.map((message) => ({ ...message, partition: this.partition(message) })), this.metadata.getTopicPartitionLeaderIds());
+        const partitionedMessages = messages.map((message) => {
+            message.partition = this.partition(message);
+            return message;
+        });
+        const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
         try {
             await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
                 const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.0.1",
+    "version": "1.0.3",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",