kafka-crab-js 1.0.1 → 1.0.3

package/README.md

@@ -2,7 +2,7 @@
 
 # 🦀 Kafka Crab JS 🦀
 
-A lightweight and flexible Kafka client for JavaScript/TypeScript, built with Rust-inspired reliability.
+A lightweight, flexible, and reliable Kafka client for JavaScript/TypeScript. It is built using Rust and librdkafka, providing a high-performance and feature-rich Kafka client.
 
 [![npm version](https://img.shields.io/npm/v/kafka-crab-js.svg)](https://www.npmjs.com/package/kafka-crab-js)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
@@ -205,12 +205,124 @@ async function produceWithMetadata() {
 }
 ```
 
+### Reconnecting Kafka Consumer
+
+```javascript
+import { KafkaClient } from 'kafka-crab-js'
+
+const kafkaClient = new KafkaClient({
+  brokers: 'localhost:29092',
+  clientId: 'reconnect-test',
+  logLevel: 'debug',
+  brokerAddressFamily: 'v4',
+  configuration: {
+    'auto.offset.reset': 'earliest',
+  },
+})
+
+/**
+ * Creates and configures a new Kafka stream consumer
+ */
+async function createConsumer() {
+  const kafkaStream = kafkaClient.createStreamConsumer({
+    groupId: 'reconnect-test',
+    enableAutoCommit: true,
+  })
+  await kafkaStream.subscribe([
+    { topic: 'foo' },
+    { topic: 'bar' },
+  ])
+  return kafkaStream
+}
+
+/**
+ * Starts a Kafka consumer with auto-restart capability
+ */
+async function startConsumer() {
+  let counter = 0
+  let retryCount = 0
+  const MAX_RETRIES = 5
+  const RETRY_DELAY = 5000 // 5 seconds
+
+  async function handleRetry() {
+    if (retryCount < MAX_RETRIES) {
+      retryCount++
+      console.log(
+        `Attempting to restart consumer (attempt ${retryCount}/${MAX_RETRIES}) in ${RETRY_DELAY / 1000} seconds...`,
+      )
+      setTimeout(setupConsumerWithRetry, RETRY_DELAY)
+    } else {
+      console.error(`Maximum retry attempts (${MAX_RETRIES}) reached. Stopping consumer.`)
+      process.exit(1)
+    }
+  }
+
+  async function setupConsumerWithRetry() {
+    try {
+      const kafkaStream = await createConsumer()
+      retryCount = 0 // Reset retry count on successful connection
+
+      console.log('Starting consumer')
+
+      kafkaStream.on('data', (message) => {
+        counter++
+        console.log('>>> Message received:', {
+          counter,
+          payload: message.payload.toString(),
+          offset: message.offset,
+          partition: message.partition,
+          topic: message.topic,
+        })
+      })
+
+      kafkaStream.on('error', async (error) => {
+        console.error('Stream error:', error)
+        handleRetry()
+      })
+
+      kafkaStream.on('close', () => {
+        console.log('Stream ended')
+        try {
+          kafkaStream.unsubscribe()
+        } catch (unsubError) {
+          console.error('Error unsubscribing:', unsubError)
+        }
+      })
+    } catch (error) {
+      console.error('Error setting up consumer:', error)
+      handleRetry()
+    }
+  }
+
+  await setupConsumerWithRetry()
+}
+
+await startConsumer()
+```
+
+### Examples
+
+You can find some examples on the [example](https://github.com/mthssdrbrg/kafka-crab-js/tree/main/example) folder of this project.
+
 ## Configuration
 
 ### Configuration properties
 
+### KafkaConfiguration
+
+| Property | Type | Default | Description |
+| --- | --- | --- | --- |
+| `brokers` | `string` || List of brokers to connect to |
+| `clientId` | `string` || Client id to use for the connection |
+| `securityProtocol` | `SecurityProtocol` || Security protocol to use (PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL) |
+| `logLevel` | `string` | `info`  | Client id to use for the connection |
+| `brokerAddressFamily` | `string` | `"v4"` | Address family to use for the connection (v4, v6) |
+| `configuration` | `Map<string, string>` | `{}` | Additional configuration options for the client. See [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) |
+
 You can see the available options here: [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
 
+
+
 ## Best Practices
 
 ### Error Handling
@@ -223,11 +335,6 @@ You can see the available options here: [librdkafka](https://github.com/edenhill
 - Configure appropriate batch sizes and compression
 - Monitor and tune consumer group performance
 
-### Resource Management
-- Implement proper shutdown procedures
-- Clean up resources (disconnect producers/consumers)
-- Handle process signals (SIGTERM, SIGINT)
-
 ### Message Processing
 - Validate message formats before processing
 - Implement proper serialization/deserialization

package/index.d.ts

@@ -1,45 +1,4 @@
-/**
- * KafkaClient class
- */
-export class KafkaClient {
-    /**
-     * Creates a KafkaClient instance
-     * @param {KafkaConfiguration} config - The Kafka configuration object
-     * @throws {Error} If the configuration is invalid
-     */
-    constructor(config: KafkaConfiguration);
-    kafkaConfiguration: KafkaConfiguration;
-    kafkaClientConfig: KafkaClientConfig;
-    /**
-     * Creates a KafkaProducer instance
-     * @param {ProducerConfiguration} [producerConfiguration] - Optional producer configuration
-     * @returns {KafkaProducer} A KafkaProducer instance
-     */
-    createProducer(producerConfiguration?: ProducerConfiguration | undefined): KafkaProducer;
-    /**
-     * Creates a KafkaConsumer instance
-     * @param {ConsumerConfiguration} consumerConfiguration - Consumer configuration
-     * @returns {KafkaConsumer} A KafkaConsumer instance
-     * @throws {Error} If the configuration is invalid
-     */
-    createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer;
-    /**
-     * Creates a KafkaStreamReadable instance
-     * @param {ConsumerConfiguration} consumerConfiguration - Consumer configuration
-     * @returns {KafkaStreamReadable} A KafkaStreamReadable instance
-     * @throws {Error} If the configuration is invalid
-     */
-    createStreamConsumer(consumerConfiguration: ConsumerConfiguration): KafkaStreamReadable;
-}
-import { KafkaStreamReadable } from "./kafka-stream-readable";
-import { ProducerConfiguration } from "./js-binding";
-import { ConsumerConfiguration } from "./js-binding";
-import { KafkaConfiguration } from "./js-binding";
-import { KafkaConsumer } from "./js-binding";
-import { CommitMode } from "./js-binding";
-import { PartitionPosition } from "./js-binding";
-import { SecurityProtocol } from "./js-binding";
-import { KafkaClientConfig } from "./js-binding";
-import { TopicPartitionConfig } from "./js-binding";
-import { KafkaProducer } from "./js-binding";
-export { KafkaStreamReadable, ProducerConfiguration, ConsumerConfiguration, KafkaConfiguration, KafkaConsumer, CommitMode, PartitionPosition, SecurityProtocol, KafkaClientConfig, TopicPartitionConfig, KafkaProducer };
+export type { ConsumerConfiguration, KafkaConfiguration, KafkaCrabError, Message, MessageProducer, OffsetModel, PartitionOffset, ProducerConfiguration, ProducerRecord, RecordMetadata, RetryStrategy, TopicPartitionConfig, } from './js-binding';
+export { CommitMode, KafkaClientConfig, KafkaConsumer, KafkaProducer, PartitionPosition, SecurityProtocol, } from './js-binding';
+export { KafkaClient } from './kafka-client';
+export { KafkaStreamReadable } from './kafka-stream-readable';

package/index.js

@@ -1,78 +1,14 @@
-const { Readable } = require('stream')
-
-const {
-  ProducerConfiguration,
-  ConsumerConfiguration,
-  KafkaConfiguration,
-  KafkaConsumer,
-  CommitMode,
-  PartitionPosition,
-  SecurityProtocol,
-  KafkaClientConfig,
-  TopicPartitionConfig,
-  KafkaProducer,
-} = require('./js-binding')
-
-const { KafkaStreamReadable } = require('./kafka-stream-readable')
-
-/**
- * KafkaClient class
- */
-class KafkaClient {
-  /**
-   * Creates a KafkaClient instance
-   * @param {KafkaConfiguration} config - The Kafka configuration object
-   * @throws {Error} If the configuration is invalid
-   */
-  constructor(config) {
-    this.kafkaConfiguration = config
-    this.kafkaClientConfig = new KafkaClientConfig(config)
-  }
-
-  /**
-   * Creates a KafkaProducer instance
-   * @param {ProducerConfiguration} [producerConfiguration] - Optional producer configuration
-   * @returns {KafkaProducer} A KafkaProducer instance
-   */
-  createProducer(producerConfiguration) {
-    if (producerConfiguration) {
-      return this.kafkaClientConfig.createProducer(producerConfiguration)
-    }
-    return this.kafkaClientConfig.createProducer({})
-  }
-
-  /**
-   * Creates a KafkaConsumer instance
-   * @param {ConsumerConfiguration} consumerConfiguration - Consumer configuration
-   * @returns {KafkaConsumer} A KafkaConsumer instance
-   * @throws {Error} If the configuration is invalid
-   */
-  createConsumer(consumerConfiguration) {
-    return this.kafkaClientConfig.createConsumer(consumerConfiguration)
-  }
-
-  /**
-   * Creates a KafkaStreamReadable instance
-   * @param {ConsumerConfiguration} consumerConfiguration - Consumer configuration
-   * @returns {KafkaStreamReadable} A KafkaStreamReadable instance
-   * @throws {Error} If the configuration is invalid
-   */
-  createStreamConsumer(consumerConfiguration) {
-    return new KafkaStreamReadable(this.kafkaClientConfig.createConsumer(consumerConfiguration))
-  }
-}
-
-module.exports = {
-  KafkaClient,
-  KafkaStreamReadable,
-  ProducerConfiguration,
-  ConsumerConfiguration,
-  KafkaConfiguration,
-  KafkaConsumer,
-  CommitMode,
-  PartitionPosition,
-  SecurityProtocol,
-  KafkaClientConfig,
-  TopicPartitionConfig,
-  KafkaProducer,
-}
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaStreamReadable = exports.KafkaClient = exports.SecurityProtocol = exports.PartitionPosition = exports.KafkaProducer = exports.KafkaConsumer = exports.KafkaClientConfig = exports.CommitMode = void 0;
+var js_binding_1 = require("./js-binding");
+Object.defineProperty(exports, "CommitMode", { enumerable: true, get: function () { return js_binding_1.CommitMode; } });
+Object.defineProperty(exports, "KafkaClientConfig", { enumerable: true, get: function () { return js_binding_1.KafkaClientConfig; } });
+Object.defineProperty(exports, "KafkaConsumer", { enumerable: true, get: function () { return js_binding_1.KafkaConsumer; } });
+Object.defineProperty(exports, "KafkaProducer", { enumerable: true, get: function () { return js_binding_1.KafkaProducer; } });
+Object.defineProperty(exports, "PartitionPosition", { enumerable: true, get: function () { return js_binding_1.PartitionPosition; } });
+Object.defineProperty(exports, "SecurityProtocol", { enumerable: true, get: function () { return js_binding_1.SecurityProtocol; } });
+var kafka_client_1 = require("./kafka-client");
+Object.defineProperty(exports, "KafkaClient", { enumerable: true, get: function () { return kafka_client_1.KafkaClient; } });
+var kafka_stream_readable_1 = require("./kafka-stream-readable");
+Object.defineProperty(exports, "KafkaStreamReadable", { enumerable: true, get: function () { return kafka_stream_readable_1.KafkaStreamReadable; } });

package/kafka-client.d.ts

@@ -0,0 +1,34 @@
+import { ConsumerConfiguration, KafkaConfiguration, ProducerConfiguration } from './js-binding.js';
+import { KafkaStreamReadable } from './kafka-stream-readable';
+/**
+ * KafkaClient class
+ */
+export declare class KafkaClient {
+    private readonly kafkaConfiguration;
+    private readonly kafkaClientConfig;
+    /**
+     * Creates a KafkaClient instance
+     * @throws {Error} If the configuration is invalid
+     */
+    constructor(kafkaConfiguration: KafkaConfiguration);
+    /**
+     * Creates a KafkaProducer instance
+     * @param {ProducerConfiguration} [producerConfiguration] - Optional producer configuration
+     * @returns {KafkaProducer} A KafkaProducer instance
+     */
+    createProducer(producerConfiguration: ProducerConfiguration): import("./js-binding.js").KafkaProducer;
+    /**
+     * Creates a KafkaConsumer instance
+     * @param {ConsumerConfiguration} consumerConfiguration - Consumer configuration
+     * @returns {KafkaConsumer} A KafkaConsumer instance
+     * @throws {Error} If the configuration is invalid
+     */
+    createConsumer(consumerConfiguration: ConsumerConfiguration): import("./js-binding.js").KafkaConsumer;
+    /**
+     * Creates a KafkaStreamReadable instance
+     * @param {ConsumerConfiguration} consumerConfiguration - Consumer configuration
+     * @returns {KafkaStreamReadable} A KafkaStreamReadable instance
+     * @throws {Error} If the configuration is invalid
+     */
+    createStreamConsumer(consumerConfiguration: ConsumerConfiguration): KafkaStreamReadable;
+}

package/kafka-client.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaClient = void 0;
+const js_binding_js_1 = require("./js-binding.js");
+const kafka_stream_readable_1 = require("./kafka-stream-readable");
+/**
+ * KafkaClient class
+ */
+class KafkaClient {
+    /**
+     * Creates a KafkaClient instance
+     * @throws {Error} If the configuration is invalid
+     */
+    constructor(kafkaConfiguration) {
+        this.kafkaConfiguration = kafkaConfiguration;
+        this.kafkaClientConfig = new js_binding_js_1.KafkaClientConfig(kafkaConfiguration);
+    }
+    /**
+     * Creates a KafkaProducer instance
+     * @param {ProducerConfiguration} [producerConfiguration] - Optional producer configuration
+     * @returns {KafkaProducer} A KafkaProducer instance
+     */
+    createProducer(producerConfiguration) {
+        if (producerConfiguration) {
+            return this.kafkaClientConfig.createProducer(producerConfiguration);
+        }
+        return this.kafkaClientConfig.createProducer({});
+    }
+    /**
+     * Creates a KafkaConsumer instance
+     * @param {ConsumerConfiguration} consumerConfiguration - Consumer configuration
+     * @returns {KafkaConsumer} A KafkaConsumer instance
+     * @throws {Error} If the configuration is invalid
+     */
+    createConsumer(consumerConfiguration) {
+        return this.kafkaClientConfig.createConsumer(consumerConfiguration);
+    }
+    /**
+     * Creates a KafkaStreamReadable instance
+     * @param {ConsumerConfiguration} consumerConfiguration - Consumer configuration
+     * @returns {KafkaStreamReadable} A KafkaStreamReadable instance
+     * @throws {Error} If the configuration is invalid
+     */
+    createStreamConsumer(consumerConfiguration) {
+        return new kafka_stream_readable_1.KafkaStreamReadable(this.kafkaClientConfig.createConsumer(consumerConfiguration));
+    }
+}
+exports.KafkaClient = KafkaClient;

package/kafka-stream-readable.d.ts

@@ -0,0 +1,31 @@
+import { Readable } from 'stream';
+import { KafkaConsumer, TopicPartitionConfig } from './js-binding';
+/**
+ * KafkaStreamReadable class
+ * @extends Readable
+ */
+export declare class KafkaStreamReadable extends Readable {
+    private readonly kafkaConsumer;
+    /**
+     * Creates a KafkaStreamReadable instance
+     */
+    constructor(kafkaConsumer: KafkaConsumer);
+    /**
+     * Subscribes to topics
+     */
+    subscribe(topics: string | Array<TopicPartitionConfig>): Promise<void>;
+    /**
+     * Unsubscribe from topics
+     */
+    unsubscribe(): void;
+    /**
+     * Returns the raw Kafka consumer
+     * @returns {KafkaConsumer} The Kafka consumer instance
+     */
+    rawConsumer(): KafkaConsumer;
+    /**
+     * Internal method called by the Readable stream to fetch data
+     * @private
+     */
+    _read(): Promise<void>;
+}

package/kafka-stream-readable.js

@@ -1,73 +1,67 @@
-const { Readable } = require('stream')
-
-const {
-  KafkaConsumer,
-  TopicPartitionConfig,
-} = require('./js-binding')
-
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaStreamReadable = void 0;
+const stream_1 = require("stream");
 /**
  * KafkaStreamReadable class
  * @extends Readable
  */
-class KafkaStreamReadable extends Readable {
-  /**
-   * Creates a KafkaStreamReadable instance
-   * @param { KafkaConsumer } kafkaConsumer - The Kafka consumer instance
-   */
-  constructor(kafkaConsumer) {
-    super({ objectMode: true })
-    if (!kafkaConsumer) {
-      throw new Error('A valid KafkaConsumer instance is required.')
+class KafkaStreamReadable extends stream_1.Readable {
+    /**
+     * Creates a KafkaStreamReadable instance
+     */
+    constructor(kafkaConsumer) {
+        super({ objectMode: true });
+        this.kafkaConsumer = kafkaConsumer;
+        if (!kafkaConsumer) {
+            throw new Error('A valid KafkaConsumer instance is required.');
+        }
+        this.kafkaConsumer = kafkaConsumer;
     }
-    this.kafkaConsumer = kafkaConsumer
-  }
-
-  /**
-   * Subscribes to topics
-   * @param {string | Array<TopicPartitionConfig>} topics  - Topics to subscribe to
-   * @returns {Promise<void>} - A promise that resolves when the subscription is successful
-   */
-  async subscribe(topics) {
-    if (!topics || (Array.isArray(topics) && topics.length === 0)) {
-      throw new Error('Topics must be a non-empty string or array.')
+    /**
+     * Subscribes to topics
+     */
+    async subscribe(topics) {
+        if (!topics || (Array.isArray(topics) && topics.length === 0)) {
+            throw new Error('Topics must be a non-empty string or array.');
+        }
+        await this.kafkaConsumer.subscribe(topics);
     }
-    await this.kafkaConsumer.subscribe(topics)
-  }
-
-  /**
-   * Unsubscribe from topics
-   * @returns {void}
-   */
-  unsubscribe() {
-    this.kafkaConsumer.unsubscribe()
-  }
-
-  /**
-   * Returns the raw Kafka consumer
-   * @returns {KafkaConsumer} The Kafka consumer instance
-   */
-  rawConsumer() {
-    return this.kafkaConsumer
-  }
-
-  /**
-   * Internal method called by the Readable stream to fetch data
-   * @private
-   */
-  async _read() {
-    try {
-      const message = await this.kafkaConsumer.recv() // Call the napi-rs method
-      if (message) {
-        this.push(message) // Push message into the stream
-      } else {
-        this.push(null) // No more data, end of stream
-      }
-    } catch (error) {
-      this.destroy(error)
+    /**
+     * Unsubscribe from topics
+     */
+    unsubscribe() {
+        this.kafkaConsumer.unsubscribe();
+    }
+    /**
+     * Returns the raw Kafka consumer
+     * @returns {KafkaConsumer} The Kafka consumer instance
+     */
+    rawConsumer() {
+        return this.kafkaConsumer;
+    }
+    /**
+     * Internal method called by the Readable stream to fetch data
+     * @private
+     */
+    async _read() {
+        try {
+            const message = await this.kafkaConsumer.recv(); // Call the napi-rs method
+            if (message) {
+                this.push(message); // Push message into the stream
+            }
+            else {
+                this.push(null); // No more data, end of stream
+            }
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                this.destroy(error);
+            }
+            else {
+                this.destroy();
+            }
+        }
     }
-  }
-}
-
-module.exports = {
-  KafkaStreamReadable,
 }
+exports.KafkaStreamReadable = KafkaStreamReadable;

package/package.json

@@ -1,13 +1,16 @@
 {
   "name": "kafka-crab-js",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "main": "index.js",
   "types": "index.d.ts",
   "module": "commonjs",
   "files": [
     "index.d.ts",
     "index.js",
+    "kafka-client.js",
+    "kafka-client.d.ts",
     "kafka-stream-readable.js",
+    "kafka-stream-readable.d.ts",
     "js-binding.js",
     "js-binding.d.ts"
   ],
@@ -43,15 +46,18 @@
     "ava": "6.2.0",
     "dprint": "^0.47.5",
     "nanoid": "5.0.8",
-    "typescript": "5.6.3"
+    "typescript": "5.6.3",
+    "copyfiles": "^2.4.1",
+    "@types/copyfiles": "^2.4.4",
+    "rimraf": "^6.0.1"
   },
   "ava": {
     "timeout": "3m"
   },
   "scripts": {
     "artifacts": "napi artifacts",
-    "build": "napi build --js js-binding.js --dts js-binding.d.ts --no-const-enum --platform --release",
-    "postbuild": "tsc",
+    "build": "napi build --js js-src/js-binding.js --dts js-src/js-binding.d.ts --no-const-enum --platform --release",
+    "postbuild": "rimraf dist && tsc && copyfiles -f js-src/js-binding.d.ts js-src/js-binding.js ./dist/*.js ./dist/*.d.ts .",
     "build:debug": "napi build --platform",
     "prepublishOnly": "napi prepublish -t npm",
     "test": "ava",
@@ -61,8 +67,8 @@
   },
   "packageManager": "pnpm@9.14.2",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "1.0.1",
-    "kafka-crab-js-darwin-x64": "1.0.1",
-    "kafka-crab-js-linux-x64-gnu": "1.0.1"
+    "kafka-crab-js-darwin-arm64": "1.0.3",
+    "kafka-crab-js-darwin-x64": "1.0.3",
+    "kafka-crab-js-linux-x64-gnu": "1.0.3"
   }
 }