kafka-ts 0.0.14 → 0.0.17-beta.0

package/README.md

@@ -38,7 +38,7 @@ const consumer = await kafka.startConsumer({
 ```typescript
 export const producer = kafka.createProducer();
 
-await producer.send([{ topic: 'my-topic', partition: 0, key: 'key', value: 'value' }]);
+await producer.send([{ topic: 'my-topic', key: 'key', value: 'value' }]);
 ```
 
 #### Low-level API
@@ -116,6 +116,26 @@ After each batch is processed, the consumer will commit offsets for the processe
 
 `concurrency` controls how many aforementioned batches are processed concurrently. 
 
+#### Partitioning
+
+By default, messages are partitioned by message key or round-robin if the key is null or undefined. Partition can be overwritten by `partition` property in the message. You can also override the default partitioner per producer instance `kafka.createProducer({ partitioner: customPartitioner })`.
+
+A simple example how to partition messages by the value in message header `x-partition-key`:
+
+```typescript
+import type { Partitioner } from 'kafka-ts';
+import { defaultPartitioner } from 'kafka-ts';
+
+const myPartitioner: Partitioner = (context) => {
+    const partition = defaultPartitioner(context);
+    return (message) => partition({ ...message, key: message.headers?.['x-partition-key'] });
+};
+
+const producer = kafka.createProducer({ partitioner: myPartitioner });
+
+await producer.send([{ topic: 'my-topic', value: 'value', headers: { 'x-partition-key': '123' } }]);
+```
+
 ## Motivation
 
 The existing low-level libraries (e.g. node-rdkafka) are bindings on librdkafka, which doesn't give enough control over the consumer logic.

package/dist/cluster.js

@@ -27,7 +27,14 @@ class Cluster {
         if (!this.seedBroker) {
             return this.connect();
         }
-        await Promise.all([this.seedBroker, ...Object.values(this.brokerById)].map((x) => x.ensureConnected()));
+        try {
+            await Promise.all([this.seedBroker, ...Object.values(this.brokerById)].map((x) => x.ensureConnected()));
+        }
+        catch {
+            logger_1.log.warn('Failed to connect to known brokers, reconnecting...');
+            await this.disconnect();
+            return this.connect();
+        }
     }
     async disconnect() {
         await Promise.all([

package/dist/connection.js

@@ -84,7 +84,7 @@ class Connection {
     disconnect() {
         this.socket.removeAllListeners();
         return new Promise((resolve) => {
-            if (this.socket.pending) {
+            if (!this.isConnected()) {
                 return resolve();
             }
             this.socket.end(resolve);
@@ -122,6 +122,7 @@ class Connection {
         }
         catch (error) {
             if (error instanceof error_1.KafkaTSApiError) {
+                error.apiName = apiName;
                 error.request = body;
             }
             throw error;

package/dist/consumer/consumer.d.ts

@@ -42,6 +42,7 @@ export declare class Consumer extends EventEmitter<{
     start(): Promise<void>;
     close(force?: boolean): Promise<void>;
     private startFetchManager;
+    private waitForReassignment;
     private process;
     private fetch;
 }

package/dist/consumer/consumer.js

@@ -80,7 +80,7 @@ class Consumer extends events_1.default {
         this.stopHook = undefined;
         try {
             await this.cluster.connect();
-            await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
+            await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
             this.metadata.setAssignment(this.metadata.getTopicPartitions());
             await this.offsetManager.fetchOffsets({ fromBeginning });
             await this.consumerGroup?.init();
@@ -131,15 +131,13 @@ class Consumer extends events_1.default {
                 });
                 await this.fetchManager.start();
                 if (!nodeAssignments.length) {
-                    logger_1.log.debug('No partitions assigned. Waiting for reassignment...', { groupId });
-                    await (0, delay_1.delay)(this.options.maxWaitMs);
-                    this.consumerGroup?.handleLastHeartbeat();
+                    await this.waitForReassignment();
                 }
             }
             catch (error) {
                 await this.fetchManager?.stop();
                 if (error.errorCode === api_1.API_ERROR.REBALANCE_IN_PROGRESS) {
-                    logger_1.log.debug('Rebalance in progress...');
+                    logger_1.log.debug('Rebalance in progress...', { apiName: error.apiName, groupId });
                     continue;
                 }
                 if (error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
@@ -162,6 +160,14 @@ class Consumer extends events_1.default {
         }
         this.stopHook?.();
     }
+    async waitForReassignment() {
+        const { groupId } = this.options;
+        logger_1.log.debug('No partitions assigned. Waiting for reassignment...', { groupId });
+        while (!this.stopHook) {
+            await (0, delay_1.delay)(1000);
+            this.consumerGroup?.handleLastHeartbeat();
+        }
+    }
     async process(messages) {
         const { options } = this;
         const { retrier } = options;

package/dist/metadata.d.ts

@@ -19,6 +19,9 @@ export declare class Metadata {
         topics: string[];
         allowTopicAutoCreation: boolean;
     }): Promise<void>;
-    private fetchMetadata;
+    fetchMetadata({ topics, allowTopicAutoCreation, }: {
+        topics: string[] | null;
+        allowTopicAutoCreation: boolean;
+    }): Promise<void>;
 }
 export {};

package/dist/metadata.js

@@ -104,3 +104,9 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], Metadata.prototype, "fetchMetadataIfNecessary", null);
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Metadata.prototype, "fetchMetadata", null);

package/dist/producer/producer.js

@@ -76,7 +76,10 @@ class Producer {
                                 offsetDelta: index,
                                 key: message.key ?? null,
                                 value: message.value,
-                                headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({ key, value })),
+                                headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
+                                    key,
+                                    value,
+                                })),
                             })),
                         };
                     }),
@@ -95,7 +98,10 @@ class Producer {
             }));
         }
         catch (error) {
-            if ((error instanceof error_1.KafkaTSApiError) && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
+            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
+                await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
+            }
+            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
                 await this.initProducerId();
             }
             throw error;

package/dist/utils/error.d.ts

@@ -5,6 +5,7 @@ export declare class KafkaTSApiError<T = any> extends KafkaTSError {
     errorCode: number;
     errorMessage: string | null;
     response: T;
+    apiName: string | undefined;
     request: unknown | undefined;
     constructor(errorCode: number, errorMessage: string | null, response: T);
 }

package/dist/utils/error.js

@@ -13,6 +13,7 @@ class KafkaTSApiError extends KafkaTSError {
     errorCode;
     errorMessage;
     response;
+    apiName;
     request;
     constructor(errorCode, errorMessage, response) {
         const [errorName] = Object.entries(api_1.API_ERROR).find(([, value]) => value === errorCode) ?? ['UNKNOWN'];

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.14",
+    "version": "0.0.17-beta.0",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",