kafka-ts 0.0.17 → 1.0.0

package/README.md

@@ -27,8 +27,8 @@ export const kafka = createKafkaClient({
 const consumer = await kafka.startConsumer({
     groupId: 'my-consumer-group',
     topics: ['my-topic'],
-    onMessage: (message) => {
-        console.log(message);
+    onBatch: (messages) => {
+        console.log(messages);
     },
 });
 ```
@@ -87,7 +87,7 @@ By default KafkaTS logs out using a JSON logger. This can be globally replaced b
 
 #### Retries
 
-By default KafkaTS retries onBatch and onMessage using an exponential backoff strategy (see [src/utils/retrier.ts](./src/utils/retrier.ts)). In case of failure, processed message offsets are committed, and the consumer is restarted.
+By default KafkaTS retries `onBatch` using an exponential backoff delay up to 5 times (see [src/utils/retrier.ts](./src/utils/retrier.ts)). In case of failure the consumer is restarted.
 
 In case you want to skip failed messages or implement a DLQ-like mechanism, you can overwrite `retrier` on `startConsumer()` and execute your own logic `onFailure`.
 
@@ -100,22 +100,6 @@ await kafka.startConsumer({
 });
 ```
 
-#### Concurrency control
-
-Depending on the use case, you might want to control `concurrency` and `batchGranularity`.
-
-When subscribing to a topic, the consumer group leader will distribute all subscribed topic partitions to consumers within the group. Each consumer will then fetch messages only from partitions assigned to them. 
-
-`batchGranularity` controls how messages are split into batches from a fetch response:
-
-- **broker** - (default) all messages received from a single kafka broker will be included in a single batch.
-- **topic** - all messages received from a single broker and topic will be included in a single batch.
-- **partition** - a batch will only include messages from a single partition.
-
-After each batch is processed, the consumer will commit offsets for the processed messages. The more granual the batch is, the more often offsets are committed.
-
-`concurrency` controls how many aforementioned batches are processed concurrently. 
-
 #### Partitioning
 
 By default, messages are partitioned by message key or round-robin if the key is null or undefined. Partition can be overwritten by `partition` property in the message. You can also override the default partitioner per producer instance `kafka.createProducer({ partitioner: customPartitioner })`.
@@ -168,25 +152,23 @@ Custom SASL mechanisms can be implemented following the `SASLProvider` interface
 
 ### `kafka.startConsumer()`
 
-| Name                   | Type                                   | Required | Default                         | Description                                                                                                                                                                                                                                                                      |
-| ---------------------- | -------------------------------------- | -------- | ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| topics                 | string[]                               | true     |                                 | List of topics to subscribe to                                                                                                                                                                                                                                                   |
-| groupId                | string                                 | false    | _null_                          | Consumer group id                                                                                                                                                                                                                                                                |
-| groupInstanceId        | string                                 | false    | _null_                          | Consumer group instance id                                                                                                                                                                                                                                                       |
-| rackId                 | string                                 | false    | _null_                          | Rack id                                                                                                                                                                                                                                                                          |
-| isolationLevel         | IsolationLevel                         | false    | IsolationLevel.READ_UNCOMMITTED | Isolation level                                                                                                                                                                                                                                                                  |
-| sessionTimeoutMs       | number                                 | false    | 30000                           | Session timeout in milliseconds                                                                                                                                                                                                                                                  |
-| rebalanceTimeoutMs     | number                                 | false    | 60000                           | Rebalance timeout in milliseconds                                                                                                                                                                                                                                                |
-| maxWaitMs              | number                                 | false    | 5000                            | Fetch long poll timeout in milliseconds                                                                                                                                                                                                                                          |
-| minBytes               | number                                 | false    | 1                               | Minimum number of bytes to wait for before returning a fetch response                                                                                                                                                                                                            |
-| maxBytes               | number                                 | false    | 1_048_576                       | Maximum number of bytes to return in the fetch response                                                                                                                                                                                                                          |
-| partitionMaxBytes      | number                                 | false    | 1_048_576                       | Maximum number of bytes to return per partition in the fetch response                                                                                                                                                                                                            |
-| allowTopicAutoCreation | boolean                                | false    | false                           | Allow kafka to auto-create topic when it doesn't exist                                                                                                                                                                                                                           |
-| fromBeginning          | boolean                                | false    | false                           | Start consuming from the beginning of the topic                                                                                                                                                                                                                                  |
-| batchGranularity       | BatchGranularity                       | false    | broker                          | Controls messages split from fetch response. Also controls how often offsets are committed. **onBatch** will include messages:<br/>- **partition** - from a single batch<br/>- **topic** - from all topic partitions<br/>- **broker** - from all assignned topics and partitions |
-| concurrency            | number                                 | false    | 1                               | How many batches to process concurrently                                                                                                                                                                                                                                         |
-| onMessage              | (message: Message) => Promise<unknown> | true     |                                 | Callback executed on every message                                                                                                                                                                                                                                               |
-| onBatch                | (batch: Message[]) => Promise<unknown> | true     |                                 | Callback executed on every batch of messages (based on **batchGranuality**)                                                                                                                                                                                                      |
+| Name                   | Type                                   | Required | Default                         | Description                                                                          |
+| ---------------------- | -------------------------------------- | -------- | ------------------------------- | ------------------------------------------------------------------------------------ |
+| topics                 | string[]                               | true     |                                 | List of topics to subscribe to                                                       |
+| groupId                | string                                 | false    | _null_                          | Consumer group id                                                                    |
+| groupInstanceId        | string                                 | false    | _null_                          | Consumer group instance id                                                           |
+| rackId                 | string                                 | false    | _null_                          | Rack id                                                                              |
+| isolationLevel         | IsolationLevel                         | false    | IsolationLevel.READ_UNCOMMITTED | Isolation level                                                                      |
+| sessionTimeoutMs       | number                                 | false    | 30000                           | Session timeout in milliseconds                                                      |
+| rebalanceTimeoutMs     | number                                 | false    | 60000                           | Rebalance timeout in milliseconds                                                    |
+| maxWaitMs              | number                                 | false    | 5000                            | Fetch long poll timeout in milliseconds                                              |
+| minBytes               | number                                 | false    | 1                               | Minimum number of bytes to wait for before returning a fetch response                |
+| maxBytes               | number                                 | false    | 1_048_576                       | Maximum number of bytes to return in the fetch response                              |
+| partitionMaxBytes      | number                                 | false    | 1_048_576                       | Maximum number of bytes to return per partition in the fetch response                |
+| allowTopicAutoCreation | boolean                                | false    | false                           | Allow kafka to auto-create topic when it doesn't exist                               |
+| fromTimestamp          | bigint                                 | false    | -1                              | Start consuming messages from timestamp (-1 = latest offsets, -2 = earliest offsets) |
+| batchSize              | number                                 | false    | null                            | Maximum number of records called `onBatch`                                           |
+| onBatch                | (batch: Message[]) => Promise<unknown> | true     |                                 | Callback executed when a batch of messages is received                               |
 
 ### `kafka.createProducer()`
 
@@ -195,7 +177,7 @@ Custom SASL mechanisms can be implemented following the `SASLProvider` interface
 | allowTopicAutoCreation | boolean     | false    | false              | Allow kafka to auto-create topic when it doesn't exist                                  |
 | partitioner            | Partitioner | false    | defaultPartitioner | Custom partitioner function. By default, it uses a default java-compatible partitioner. |
 
-### `producer.send(messages: Message[])`
+### `producer.send(messages: Message[], options?: { acks?: -1 | 1 })`
 
 <!-- export type Message = {
     topic: string;

package/dist/api/fetch.d.ts

@@ -27,20 +27,25 @@ export declare const FETCH: import("../utils/api").Api<{
     }[];
     rackId: string;
 }, {
+    _tag: void;
+    throttleTimeMs: number;
+    errorCode: number;
+    sessionId: number;
     responses: {
+        topicId: string;
         partitions: {
+            partitionIndex: number;
+            errorCode: number;
+            highWatermark: bigint;
+            lastStableOffset: bigint;
+            logStartOffset: bigint;
+            abortedTransactions: {
+                producerId: bigint;
+                firstOffset: bigint;
+                _tag: void;
+            }[];
+            preferredReadReplica: number;
             records: {
-                records: {
-                    attributes: number;
-                    timestampDelta: bigint;
-                    offsetDelta: number;
-                    key: string | null;
-                    value: string | null;
-                    headers: {
-                        key: string;
-                        value: string;
-                    }[];
-                }[];
                 baseOffset: bigint;
                 batchLength: number;
                 partitionLeaderEpoch: number;
@@ -58,26 +63,21 @@ export declare const FETCH: import("../utils/api").Api<{
                 producerId: bigint;
                 producerEpoch: number;
                 baseSequence: number;
+                records: {
+                    attributes: number;
+                    timestampDelta: bigint;
+                    offsetDelta: number;
+                    key: string | null;
+                    value: string | null;
+                    headers: {
+                        key: string;
+                        value: string;
+                    }[];
+                }[];
             }[];
-            partitionIndex: number;
-            errorCode: number;
-            highWatermark: bigint;
-            lastStableOffset: bigint;
-            logStartOffset: bigint;
-            abortedTransactions: {
-                producerId: bigint;
-                firstOffset: bigint;
-                _tag: void;
-            }[];
-            preferredReadReplica: number;
             _tag: void;
         }[];
-        topicId: string;
         _tag: void;
     }[];
-    _tag: void;
-    throttleTimeMs: number;
-    errorCode: number;
-    sessionId: number;
     _tag2: void;
 }>;

package/dist/api/fetch.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.FETCH = void 0;
-const codecs_1 = require("../codecs");
 const api_1 = require("../utils/api");
 const decoder_1 = require("../utils/decoder");
 const error_1 = require("../utils/error");
@@ -69,19 +68,7 @@ exports.FETCH = (0, api_1.createApi)({
                     throw new error_1.KafkaTSApiError(partition.errorCode, null, result);
             });
         });
-        const decompressedResponses = await Promise.all(result.responses.map(async (response) => ({
-            ...response,
-            partitions: await Promise.all(response.partitions.map(async (partition) => ({
-                ...partition,
-                records: await Promise.all(partition.records.map(async ({ recordsLength, compressedRecords, ...record }) => {
-                    const { decompress } = (0, codecs_1.findCodec)(record.compression);
-                    const decompressedRecords = await decompress(compressedRecords);
-                    const decompressedDecoder = new decoder_1.Decoder(Buffer.concat([recordsLength, decompressedRecords]));
-                    return { ...record, records: decodeRecord(decompressedDecoder) };
-                })),
-            }))),
-        })));
-        return { ...result, responses: decompressedResponses };
+        return result;
     },
 });
 const decodeRecordBatch = (decoder) => {
@@ -125,8 +112,7 @@ const decodeRecordBatch = (decoder) => {
         const producerId = batchDecoder.readInt64();
         const producerEpoch = batchDecoder.readInt16();
         const baseSequence = batchDecoder.readInt32();
-        const recordsLength = batchDecoder.read(4);
-        const compressedRecords = batchDecoder.read();
+        const records = decodeRecords(batchDecoder);
         results.push({
             baseOffset,
             batchLength,
@@ -145,13 +131,12 @@ const decodeRecordBatch = (decoder) => {
             producerId,
             producerEpoch,
             baseSequence,
-            recordsLength,
-            compressedRecords,
+            records,
         });
     }
     return results;
 };
-const decodeRecord = (decoder) => decoder.readRecords((record) => ({
+const decodeRecords = (decoder) => decoder.readRecords((record) => ({
     attributes: record.readInt8(),
     timestampDelta: record.readVarLong(),
     offsetDelta: record.readVarInt(),

package/dist/api/index.d.ts

@@ -90,20 +90,25 @@ export declare const API: {
         }[];
         rackId: string;
     }, {
+        _tag: void;
+        throttleTimeMs: number;
+        errorCode: number;
+        sessionId: number;
         responses: {
+            topicId: string;
             partitions: {
+                partitionIndex: number;
+                errorCode: number;
+                highWatermark: bigint;
+                lastStableOffset: bigint;
+                logStartOffset: bigint;
+                abortedTransactions: {
+                    producerId: bigint;
+                    firstOffset: bigint;
+                    _tag: void;
+                }[];
+                preferredReadReplica: number;
                 records: {
-                    records: {
-                        attributes: number;
-                        timestampDelta: bigint;
-                        offsetDelta: number;
-                        key: string | null;
-                        value: string | null;
-                        headers: {
-                            key: string;
-                            value: string;
-                        }[];
-                    }[];
                     baseOffset: bigint;
                     batchLength: number;
                     partitionLeaderEpoch: number;
@@ -121,27 +126,22 @@ export declare const API: {
                     producerId: bigint;
                     producerEpoch: number;
                     baseSequence: number;
+                    records: {
+                        attributes: number;
+                        timestampDelta: bigint;
+                        offsetDelta: number;
+                        key: string | null;
+                        value: string | null;
+                        headers: {
+                            key: string;
+                            value: string;
+                        }[];
+                    }[];
                 }[];
-                partitionIndex: number;
-                errorCode: number;
-                highWatermark: bigint;
-                lastStableOffset: bigint;
-                logStartOffset: bigint;
-                abortedTransactions: {
-                    producerId: bigint;
-                    firstOffset: bigint;
-                    _tag: void;
-                }[];
-                preferredReadReplica: number;
                 _tag: void;
             }[];
-            topicId: string;
             _tag: void;
         }[];
-        _tag: void;
-        throttleTimeMs: number;
-        errorCode: number;
-        sessionId: number;
         _tag2: void;
     }>;
     FIND_COORDINATOR: Api<{

package/dist/consumer/consumer.d.ts

@@ -4,7 +4,6 @@ import { IsolationLevel } from '../api/fetch';
 import { Cluster } from '../cluster';
 import { Message } from '../types';
 import { Retrier } from '../utils/retrier';
-import { BatchGranularity } from './fetch-manager';
 export type ConsumerOptions = {
     topics: string[];
     groupId?: string | null;
@@ -20,14 +19,10 @@ export type ConsumerOptions = {
     allowTopicAutoCreation?: boolean;
     fromBeginning?: boolean;
     fromTimestamp?: bigint;
-    batchGranularity?: BatchGranularity;
-    concurrency?: number;
+    batchSize?: number | null;
     retrier?: Retrier;
-} & ({
     onBatch: (messages: Required<Message>[]) => unknown;
-} | {
-    onMessage: (message: Required<Message>) => unknown;
-});
+};
 export declare class Consumer extends EventEmitter<{
     offsetCommit: [];
     heartbeat: [];

package/dist/consumer/consumer.js

@@ -52,8 +52,7 @@ class Consumer extends events_1.default {
             allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
             fromBeginning: options.fromBeginning ?? false,
             fromTimestamp: options.fromTimestamp ?? (options.fromBeginning ? -2n : -1n),
-            batchGranularity: options.batchGranularity ?? 'broker',
-            concurrency: options.concurrency ?? 1,
+            batchSize: options.batchSize ?? null,
             retrier: options.retrier ?? retrier_1.defaultRetrier,
         };
         this.metadata = new consumer_metadata_1.ConsumerMetadata({ cluster: this.cluster });
@@ -107,7 +106,7 @@ class Consumer extends events_1.default {
         await this.cluster.disconnect().catch((error) => logger_1.log.debug(`Failed to disconnect: ${error.message}`));
     }
     async startFetchManager() {
-        const { groupId, batchGranularity, concurrency } = this.options;
+        const { groupId, batchSize } = this.options;
         while (!this.stopHook) {
             try {
                 await this.consumerGroup?.join();
@@ -119,16 +118,13 @@ class Consumer extends events_1.default {
                         Object.keys(partitions).map(Number),
                     ])),
                 }));
-                const numPartitions = Object.values(this.metadata.getAssignment()).flat().length;
-                const numProcessors = Math.min(concurrency, numPartitions);
                 this.fetchManager = new fetch_manager_1.FetchManager({
                     fetch: this.fetch.bind(this),
                     process: this.process.bind(this),
                     metadata: this.metadata,
                     consumerGroup: this.consumerGroup,
+                    batchSize,
                     nodeAssignments,
-                    batchGranularity,
-                    concurrency: numProcessors,
                 });
                 await this.fetchManager.start();
                 if (!nodeAssignments.length) {
@@ -177,25 +173,10 @@ class Consumer extends events_1.default {
             topicPartitions[topic] ??= new Set();
             topicPartitions[topic].add(partition);
         }
-        const commit = async () => {
-            await this.consumerGroup?.offsetCommit(topicPartitions);
-            this.offsetManager.flush(topicPartitions);
-        };
-        if ('onBatch' in options) {
-            await retrier(() => options.onBatch(messages));
-            messages.forEach(({ topic, partition, offset }) => this.offsetManager.resolve(topic, partition, offset + 1n));
-        }
-        else if ('onMessage' in options) {
-            for (const message of messages) {
-                await retrier(() => options.onMessage(message)).catch(async (error) => {
-                    await commit().catch();
-                    throw error;
-                });
-                const { topic, partition, offset } = message;
-                this.offsetManager.resolve(topic, partition, offset + 1n);
-            }
-        }
-        await commit();
+        await retrier(() => options.onBatch(messages));
+        messages.forEach(({ topic, partition, offset }) => this.offsetManager.resolve(topic, partition, offset + 1n));
+        await this.consumerGroup?.offsetCommit(topicPartitions);
+        this.offsetManager.flush(topicPartitions);
     }
     fetch(nodeId, assignment) {
         const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;

package/dist/consumer/fetch-manager.d.ts

@@ -1,33 +1,31 @@
 import { FetchResponse } from '../api/fetch';
 import { Assignment } from '../api/sync-group';
 import { Metadata } from '../metadata';
-import { Batch } from '../types';
+import { Batch, Message } from '../types';
 import { ConsumerGroup } from './consumer-group';
-export type BatchGranularity = 'partition' | 'topic' | 'broker';
 type FetchManagerOptions = {
     fetch: (nodeId: number, assignment: Assignment) => Promise<FetchResponse>;
     process: (batch: Batch) => Promise<void>;
+    batchSize?: number | null;
     metadata: Metadata;
     consumerGroup?: ConsumerGroup;
     nodeAssignments: {
         nodeId: number;
         assignment: Assignment;
     }[];
-    batchGranularity: BatchGranularity;
-    concurrency: number;
 };
 export declare class FetchManager {
     private options;
     private queue;
     private isRunning;
     private fetchers;
-    private processors;
-    private pollQueue;
+    private processor;
+    private pollCallback;
     private fetcherCallbacks;
     constructor(options: FetchManagerOptions);
     start(): Promise<void>;
     stop(): Promise<void>;
-    poll(): Promise<Batch>;
+    poll(): Promise<Required<Message>[]>;
     private onResponse;
 }
 export {};

package/dist/consumer/fetch-manager.js

@@ -10,7 +10,6 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.FetchManager = void 0;
-const error_1 = require("../utils/error");
 const tracer_1 = require("../utils/tracer");
 const fetcher_1 = require("./fetcher");
 const processor_1 = require("./processor");
@@ -20,28 +19,25 @@ class FetchManager {
     queue = [];
     isRunning = false;
     fetchers;
-    processors;
-    pollQueue = [];
+    processor;
+    pollCallback;
     fetcherCallbacks = {};
     constructor(options) {
         this.options = options;
-        const { fetch, process, nodeAssignments, concurrency } = this.options;
+        const { fetch, process, nodeAssignments } = this.options;
         this.fetchers = nodeAssignments.map(({ nodeId, assignment }, index) => new fetcher_1.Fetcher(index, {
             nodeId,
             assignment,
             fetch,
             onResponse: this.onResponse.bind(this),
         }));
-        this.processors = Array.from({ length: concurrency }).map(() => new processor_1.Processor({ process, poll: this.poll.bind(this) }));
+        this.processor = new processor_1.Processor({ process, poll: this.poll.bind(this) });
     }
     async start() {
         this.queue = [];
         this.isRunning = true;
         try {
-            await Promise.all([
-                ...this.fetchers.map((fetcher) => fetcher.loop()),
-                ...this.processors.map((processor) => processor.loop()),
-            ]);
+            await Promise.all([...this.fetchers.map((fetcher) => fetcher.loop()), this.processor.loop()]);
         }
         finally {
             await this.stop();
@@ -51,10 +47,9 @@ class FetchManager {
         this.isRunning = false;
         const stopPromise = Promise.all([
             ...this.fetchers.map((fetcher) => fetcher.stop()),
-            ...this.processors.map((processor) => processor.stop()),
+            this.processor.stop(),
         ]);
-        this.pollQueue.forEach((resolve) => resolve());
-        this.pollQueue = [];
+        this.pollCallback?.();
         Object.values(this.fetcherCallbacks).forEach((callback) => callback());
         this.fetcherCallbacks = {};
         await stopPromise;
@@ -63,35 +58,37 @@ class FetchManager {
         if (!this.isRunning) {
             return [];
         }
-        const { consumerGroup } = this.options;
+        const { consumerGroup, batchSize } = this.options;
         consumerGroup?.handleLastHeartbeat();
-        const batch = this.queue.shift();
-        if (!batch) {
-            // wait until new data is available or fetch manager is requested to stop
-            await new Promise((resolve) => {
-                this.pollQueue.push(resolve);
-            });
+        const batch = this.queue.splice(0, batchSize ?? undefined);
+        if (!batch.length) {
+            await new Promise((resolve) => (this.pollCallback = resolve));
             return this.poll();
         }
-        if ('kind' in batch && batch.kind === 'checkpoint') {
-            this.fetcherCallbacks[batch.fetcherId]?.();
-            return this.poll();
-        }
-        this.pollQueue?.shift()?.();
-        return batch;
+        const [checkpoints, messages] = partition(batch, (entry) => 'kind' in entry && entry.kind === 'checkpoint');
+        checkpoints.forEach(({ fetcherId }) => this.fetcherCallbacks[fetcherId]?.());
+        return messages;
     }
     async onResponse(fetcherId, response) {
-        const { metadata, batchGranularity, consumerGroup } = this.options;
+        const { metadata, consumerGroup } = this.options;
         consumerGroup?.handleLastHeartbeat();
-        const batches = fetchResponseToBatches(response, batchGranularity, metadata);
-        if (!batches.length) {
+        const messages = response.responses.flatMap(({ topicId, partitions }) => partitions.flatMap(({ partitionIndex, records }) => records.flatMap(({ baseTimestamp, baseOffset, records }) => records.flatMap((message) => ({
+            topic: metadata.getTopicNameById(topicId),
+            partition: partitionIndex,
+            key: message.key ?? null,
+            value: message.value ?? null,
+            headers: Object.fromEntries(message.headers.map(({ key, value }) => [key, value])),
+            timestamp: baseTimestamp + BigInt(message.timestampDelta),
+            offset: baseOffset + BigInt(message.offsetDelta),
+        })))));
+        if (!messages.length) {
             return;
         }
         // wait until all broker batches have been processed or fetch manager is requested to stop
         await new Promise((resolve) => {
             this.fetcherCallbacks[fetcherId] = resolve;
-            this.queue.push(...batches, { kind: 'checkpoint', fetcherId });
-            this.pollQueue?.shift()?.();
+            this.queue.push(...messages, { kind: 'checkpoint', fetcherId });
+            this.pollCallback?.();
         });
         consumerGroup?.handleLastHeartbeat();
     }
@@ -115,29 +112,16 @@ __decorate([
     __metadata("design:paramtypes", [Number, Object]),
     __metadata("design:returntype", Promise)
 ], FetchManager.prototype, "onResponse", null);
-const fetchResponseToBatches = (batch, batchGranularity, metadata) => {
-    const brokerTopics = batch.responses.map(({ topicId, partitions }) => partitions.map(({ partitionIndex, records }) => records.flatMap(({ baseTimestamp, baseOffset, records }) => records.map((message) => ({
-        topic: metadata.getTopicNameById(topicId),
-        partition: partitionIndex,
-        key: message.key ?? null,
-        value: message.value ?? null,
-        headers: Object.fromEntries(message.headers.map(({ key, value }) => [key, value])),
-        timestamp: baseTimestamp + BigInt(message.timestampDelta),
-        offset: baseOffset + BigInt(message.offsetDelta),
-    })))));
-    switch (batchGranularity) {
-        case 'broker':
-            const messages = brokerTopics.flatMap((topicPartition) => topicPartition.flatMap((partitionMessages) => partitionMessages));
-            return messages.length ? [messages] : [];
-        case 'topic':
-            return brokerTopics
-                .map((topicPartition) => topicPartition.flatMap((partitionMessages) => partitionMessages))
-                .filter((messages) => messages.length);
-        case 'partition':
-            return brokerTopics
-                .flatMap((topicPartition) => topicPartition.map((partitionMessages) => partitionMessages))
-                .filter((messages) => messages.length);
-        default:
-            throw new error_1.KafkaTSError(`Unhandled batch granularity: ${batchGranularity}`);
+const partition = (batch, predicate) => {
+    const checkpoints = [];
+    const messages = [];
+    for (const entry of batch) {
+        if (predicate(entry)) {
+            checkpoints.push(entry);
+        }
+        else {
+            messages.push(entry);
+        }
     }
+    return [checkpoints, messages];
 };

package/dist/utils/tracer.d.ts

@@ -1,5 +1,5 @@
 export interface Tracer {
     startActiveSpan<T>(module: string, method: string, metadata: Record<string, unknown>, callback: () => T): T;
 }
-export declare const setTracer: <T>(newTracer: Tracer) => void;
+export declare const setTracer: (newTracer: Tracer) => void;
 export declare const createTracer: (module: string) => (fn?: (...args: any[]) => Record<string, unknown> | undefined) => (target: any, propertyKey: string, descriptor: PropertyDescriptor) => void;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.17",
+    "version": "1.0.0",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",
@@ -13,6 +13,7 @@
         "down": "KAFKA_VERSION=3.7.1 docker-compose down",
         "version:prerelease": "npm version prerelease --preid=beta",
         "version:patch": "npm version patch",
+        "version:major": "npm version major",
         "format": "prettier --write .",
         "build": "tsc",
         "watch": "tsc -w",