kafka-ts 0.0.16 → 0.0.17-beta.1

package/README.md

@@ -38,7 +38,7 @@ const consumer = await kafka.startConsumer({
 ```typescript
 export const producer = kafka.createProducer();
 
-await producer.send([{ topic: 'my-topic', partition: 0, key: 'key', value: 'value' }]);
+await producer.send([{ topic: 'my-topic', key: 'key', value: 'value' }]);
 ```
 
 #### Low-level API
@@ -116,6 +116,26 @@ After each batch is processed, the consumer will commit offsets for the processe
 
 `concurrency` controls how many aforementioned batches are processed concurrently. 
 
+#### Partitioning
+
+By default, messages are partitioned by message key or round-robin if the key is null or undefined. Partition can be overwritten by `partition` property in the message. You can also override the default partitioner per producer instance `kafka.createProducer({ partitioner: customPartitioner })`.
+
+A simple example how to partition messages by the value in message header `x-partition-key`:
+
+```typescript
+import type { Partitioner } from 'kafka-ts';
+import { defaultPartitioner } from 'kafka-ts';
+
+const myPartitioner: Partitioner = (context) => {
+    const partition = defaultPartitioner(context);
+    return (message) => partition({ ...message, key: message.headers?.['x-partition-key'] });
+};
+
+const producer = kafka.createProducer({ partitioner: myPartitioner });
+
+await producer.send([{ topic: 'my-topic', value: 'value', headers: { 'x-partition-key': '123' } }]);
+```
+
 ## Motivation
 
 The existing low-level libraries (e.g. node-rdkafka) are bindings on librdkafka, which doesn't give enough control over the consumer logic.

package/dist/api/create-topics.d.ts

@@ -1,19 +1,19 @@
 export declare const CREATE_TOPICS: import("../utils/api").Api<{
     topics: {
         name: string;
-        numPartitions: number;
-        replicationFactor: number;
-        assignments: {
+        numPartitions?: number;
+        replicationFactor?: number;
+        assignments?: {
             partitionIndex: number;
             brokerIds: number[];
         }[];
-        configs: {
+        configs?: {
             name: string;
             value: string | null;
         }[];
     }[];
-    timeoutMs: number;
-    validateOnly: boolean;
+    timeoutMs?: number | undefined;
+    validateOnly?: boolean | undefined;
 }, {
     _tag: void;
     throttleTimeMs: number;

package/dist/api/create-topics.js

@@ -10,16 +10,16 @@ exports.CREATE_TOPICS = (0, api_1.createApi)({
         .writeUVarInt(0)
         .writeCompactArray(data.topics, (encoder, topic) => encoder
         .writeCompactString(topic.name)
-        .writeInt32(topic.numPartitions)
-        .writeInt16(topic.replicationFactor)
-        .writeCompactArray(topic.assignments, (encoder, assignment) => encoder
+        .writeInt32(topic.numPartitions ?? -1)
+        .writeInt16(topic.replicationFactor ?? -1)
+        .writeCompactArray(topic.assignments ?? [], (encoder, assignment) => encoder
         .writeInt32(assignment.partitionIndex)
         .writeCompactArray(assignment.brokerIds, (encoder, brokerId) => encoder.writeInt32(brokerId))
         .writeUVarInt(0))
-        .writeCompactArray(topic.configs, (encoder, config) => encoder.writeCompactString(config.name).writeCompactString(config.value).writeUVarInt(0))
+        .writeCompactArray(topic.configs ?? [], (encoder, config) => encoder.writeCompactString(config.name).writeCompactString(config.value).writeUVarInt(0))
         .writeUVarInt(0))
-        .writeInt32(data.timeoutMs)
-        .writeBoolean(data.validateOnly)
+        .writeInt32(data.timeoutMs ?? 10_000)
+        .writeBoolean(data.validateOnly ?? false)
         .writeUVarInt(0),
     response: (decoder) => {
         const result = {

package/dist/api/delete-topics.d.ts

@@ -3,7 +3,7 @@ export declare const DELETE_TOPICS: import("../utils/api").Api<{
         name: string | null;
         topicId: string | null;
     }[];
-    timeoutMs: number;
+    timeoutMs?: number | undefined;
 }, {
     _tag: void;
     throttleTimeMs: number;

package/dist/api/delete-topics.js

@@ -9,7 +9,7 @@ exports.DELETE_TOPICS = (0, api_1.createApi)({
     request: (encoder, data) => encoder
         .writeUVarInt(0)
         .writeCompactArray(data.topics, (encoder, topic) => encoder.writeCompactString(topic.name).writeUUID(topic.topicId).writeUVarInt(0))
-        .writeInt32(data.timeoutMs)
+        .writeInt32(data.timeoutMs ?? 10_000)
         .writeUVarInt(0),
     response: (decoder) => {
         const result = {

package/dist/api/index.d.ts

@@ -13,19 +13,19 @@ export declare const API: {
     CREATE_TOPICS: Api<{
         topics: {
             name: string;
-            numPartitions: number;
-            replicationFactor: number;
-            assignments: {
+            numPartitions?: number | undefined;
+            replicationFactor?: number | undefined;
+            assignments?: {
                 partitionIndex: number;
                 brokerIds: number[];
-            }[];
-            configs: {
+            }[] | undefined;
+            configs?: {
                 name: string;
                 value: string | null;
-            }[];
+            }[] | undefined;
         }[];
-        timeoutMs: number;
-        validateOnly: boolean;
+        timeoutMs?: number | undefined;
+        validateOnly?: boolean | undefined;
     }, {
         _tag: void;
         throttleTimeMs: number;
@@ -53,7 +53,7 @@ export declare const API: {
             name: string | null;
             topicId: string | null;
         }[];
-        timeoutMs: number;
+        timeoutMs?: number | undefined;
     }, {
         _tag: void;
         throttleTimeMs: number;
@@ -266,12 +266,12 @@ export declare const API: {
         _tag2: void;
     }>;
     METADATA: Api<{
-        topics: {
+        topics?: {
             id: string | null;
             name: string;
-        }[] | null;
-        allowTopicAutoCreation: boolean;
-        includeTopicAuthorizedOperations: boolean;
+        }[] | null | undefined;
+        allowTopicAutoCreation?: boolean | undefined;
+        includeTopicAuthorizedOperations?: boolean | undefined;
     }, {
         _tag: void;
         throttleTimeMs: number;

package/dist/api/metadata.d.ts

@@ -1,11 +1,11 @@
 export type Metadata = Awaited<ReturnType<(typeof METADATA)['response']>>;
 export declare const METADATA: import("../utils/api").Api<{
-    topics: {
+    topics?: {
         id: string | null;
         name: string;
-    }[] | null;
-    allowTopicAutoCreation: boolean;
-    includeTopicAuthorizedOperations: boolean;
+    }[] | null | undefined;
+    allowTopicAutoCreation?: boolean | undefined;
+    includeTopicAuthorizedOperations?: boolean | undefined;
 }, {
     _tag: void;
     throttleTimeMs: number;

package/dist/api/metadata.js

@@ -8,9 +8,9 @@ exports.METADATA = (0, api_1.createApi)({
     apiVersion: 12,
     request: (encoder, data) => encoder
         .writeUVarInt(0)
-        .writeCompactArray(data.topics, (encoder, topic) => encoder.writeUUID(topic.id).writeCompactString(topic.name).writeUVarInt(0))
-        .writeBoolean(data.allowTopicAutoCreation)
-        .writeBoolean(data.includeTopicAuthorizedOperations)
+        .writeCompactArray(data.topics ?? null, (encoder, topic) => encoder.writeUUID(topic.id).writeCompactString(topic.name).writeUVarInt(0))
+        .writeBoolean(data.allowTopicAutoCreation ?? false)
+        .writeBoolean(data.includeTopicAuthorizedOperations ?? false)
         .writeUVarInt(0),
     response: (decoder) => {
         const result = {

package/dist/cluster.js

@@ -16,18 +16,21 @@ class Cluster {
     async connect() {
         this.seedBroker = await this.findSeedBroker();
         this.brokerById = {};
-        const metadata = await this.sendRequest(api_1.API.METADATA, {
-            allowTopicAutoCreation: false,
-            includeTopicAuthorizedOperations: false,
-            topics: [],
-        });
+        const metadata = await this.sendRequest(api_1.API.METADATA, { topics: [] });
         this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
     }
     async ensureConnected() {
         if (!this.seedBroker) {
             return this.connect();
         }
-        await Promise.all([this.seedBroker, ...Object.values(this.brokerById)].map((x) => x.ensureConnected()));
+        try {
+            await Promise.all([this.seedBroker, ...Object.values(this.brokerById)].map((x) => x.ensureConnected()));
+        }
+        catch {
+            logger_1.log.warn('Failed to connect to known brokers, reconnecting...');
+            await this.disconnect();
+            return this.connect();
+        }
     }
     async disconnect() {
         await Promise.all([

package/dist/connection.js

@@ -84,7 +84,7 @@ class Connection {
     disconnect() {
         this.socket.removeAllListeners();
         return new Promise((resolve) => {
-            if (this.socket.pending) {
+            if (!this.isConnected()) {
                 return resolve();
             }
             this.socket.end(resolve);

package/dist/consumer/consumer.js

@@ -80,7 +80,7 @@ class Consumer extends events_1.default {
         this.stopHook = undefined;
         try {
             await this.cluster.connect();
-            await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
+            await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
             this.metadata.setAssignment(this.metadata.getTopicPartitions());
             await this.offsetManager.fetchOffsets({ fromBeginning });
             await this.consumerGroup?.init();
@@ -137,7 +137,7 @@ class Consumer extends events_1.default {
             catch (error) {
                 await this.fetchManager?.stop();
                 if (error.errorCode === api_1.API_ERROR.REBALANCE_IN_PROGRESS) {
-                    logger_1.log.debug('Rebalance in progress...', { apiName: error.apiName });
+                    logger_1.log.debug('Rebalance in progress...', { apiName: error.apiName, groupId });
                     continue;
                 }
                 if (error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {

package/dist/metadata.d.ts

@@ -19,6 +19,9 @@ export declare class Metadata {
         topics: string[];
         allowTopicAutoCreation: boolean;
     }): Promise<void>;
-    private fetchMetadata;
+    fetchMetadata({ topics, allowTopicAutoCreation, }: {
+        topics: string[] | null;
+        allowTopicAutoCreation: boolean;
+    }): Promise<void>;
 }
 export {};

package/dist/metadata.js

@@ -63,7 +63,6 @@ class Metadata {
         const { cluster } = this.options;
         const response = await cluster.sendRequest(api_1.API.METADATA, {
             allowTopicAutoCreation,
-            includeTopicAuthorizedOperations: false,
             topics: topics?.map((name) => ({ id: null, name })) ?? null,
         });
         this.topicPartitions = {
@@ -104,3 +103,9 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], Metadata.prototype, "fetchMetadataIfNecessary", null);
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Metadata.prototype, "fetchMetadata", null);

package/dist/producer/producer.js

@@ -76,7 +76,10 @@ class Producer {
                                 offsetDelta: index,
                                 key: message.key ?? null,
                                 value: message.value,
-                                headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({ key, value })),
+                                headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
+                                    key,
+                                    value,
+                                })),
                             })),
                         };
                     }),
@@ -95,7 +98,10 @@ class Producer {
             }));
         }
         catch (error) {
-            if ((error instanceof error_1.KafkaTSApiError) && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
+            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
+                await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
+            }
+            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
                 await this.initProducerId();
             }
             throw error;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.16",
+    "version": "0.0.17-beta.1",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",