kafka-crab-js 1.0.2 → 1.0.5

package/README.md

@@ -2,7 +2,7 @@
 
 # 🦀 Kafka Crab JS 🦀
 
-A lightweight and flexible Kafka client for JavaScript/TypeScript, built with Rust-inspired reliability.
+A lightweight, flexible, and reliable Kafka client for JavaScript/TypeScript. It is built using Rust and librdkafka, providing a high-performance and feature-rich Kafka client.
 
 [![npm version](https://img.shields.io/npm/v/kafka-crab-js.svg)](https://www.npmjs.com/package/kafka-crab-js)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
@@ -38,6 +38,8 @@ A lightweight and flexible Kafka client for JavaScript/TypeScript, built with Ru
 npm install kafka-crab-js
 # or
 yarn add kafka-crab-js
+# or
+pnpm add kafka-crab-js
 ```
 
 ## Quick Start
@@ -205,11 +207,123 @@ async function produceWithMetadata() {
 }
 ```
 
+### Reconnecting Kafka Consumer
+
+```javascript
+import { KafkaClient } from 'kafka-crab-js'
+
+const kafkaClient = new KafkaClient({
+  brokers: 'localhost:29092',
+  clientId: 'reconnect-test',
+  logLevel: 'debug',
+  brokerAddressFamily: 'v4',
+  configuration: {
+    'auto.offset.reset': 'earliest',
+  },
+})
+
+/**
+ * Creates and configures a new Kafka stream consumer
+ */
+async function createConsumer() {
+  const kafkaStream = kafkaClient.createStreamConsumer({
+    groupId: 'reconnect-test',
+    enableAutoCommit: true,
+  })
+  await kafkaStream.subscribe([
+    { topic: 'foo' },
+    { topic: 'bar' },
+  ])
+  return kafkaStream
+}
+
+/**
+ * Starts a Kafka consumer with auto-restart capability
+ */
+async function startConsumer() {
+  let counter = 0
+  let retryCount = 0
+  const MAX_RETRIES = 5
+  const RETRY_DELAY = 5000 // 5 seconds
+
+  async function handleRetry() {
+    if (retryCount < MAX_RETRIES) {
+      retryCount++
+      console.log(
+        `Attempting to restart consumer (attempt ${retryCount}/${MAX_RETRIES}) in ${RETRY_DELAY / 1000} seconds...`,
+      )
+      setTimeout(setupConsumerWithRetry, RETRY_DELAY)
+    } else {
+      console.error(`Maximum retry attempts (${MAX_RETRIES}) reached. Stopping consumer.`)
+      process.exit(1)
+    }
+  }
+
+  async function setupConsumerWithRetry() {
+    try {
+      const kafkaStream = await createConsumer()
+      retryCount = 0 // Reset retry count on successful connection
+
+      console.log('Starting consumer')
+
+      kafkaStream.on('data', (message) => {
+        counter++
+        console.log('>>> Message received:', {
+          counter,
+          payload: message.payload.toString(),
+          offset: message.offset,
+          partition: message.partition,
+          topic: message.topic,
+        })
+      })
+
+      kafkaStream.on('error', async (error) => {
+        console.error('Stream error:', error)
+        handleRetry()
+      })
+
+      kafkaStream.on('close', () => {
+        console.log('Stream ended')
+        try {
+          kafkaStream.unsubscribe()
+        } catch (unsubError) {
+          console.error('Error unsubscribing:', unsubError)
+        }
+      })
+    } catch (error) {
+      console.error('Error setting up consumer:', error)
+      handleRetry()
+    }
+  }
+
+  await setupConsumerWithRetry()
+}
+
+await startConsumer()
+```
+
+### Examples
+
+You can find some examples on the [example](https://github.com/mthssdrbrg/kafka-crab-js/tree/main/example) folder of this project.
+
 ## Configuration
 
 ### Configuration properties
 
-You can see the available options here: [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
+### KafkaConfiguration
+
+| Property | Type | Default | Description |
+| --- | --- | --- | --- |
+| `brokers` | `string` || List of brokers to connect to |
+| `clientId` | `string` || Client id to use for the connection |
+| `securityProtocol` | `SecurityProtocol` || Security protocol to use (PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL) |
+| `logLevel` | `string` | `info`  | Client id to use for the connection |
+| `brokerAddressFamily` | `string` | `"v4"` | Address family to use for the connection (v4, v6) |
+| `configuration` | `Map<string, string>` | `{}` | Additional configuration options for the client. See [librdkafka](https://docs.confluent.io/platform/current/clients/librdkafka/html/md_CONFIGURATION.html) |
+
+You can see the available options here: [librdkafka](https://docs.confluent.io/platform/current/clients/librdkafka/html/md_CONFIGURATION.html).
+
+
 
 ## Best Practices
 
@@ -223,11 +337,6 @@ You can see the available options here: [librdkafka](https://github.com/edenhill
 - Configure appropriate batch sizes and compression
 - Monitor and tune consumer group performance
 
-### Resource Management
-- Implement proper shutdown procedures
-- Clean up resources (disconnect producers/consumers)
-- Handle process signals (SIGTERM, SIGINT)
-
 ### Message Processing
 - Validate message formats before processing
 - Implement proper serialization/deserialization

package/js-binding.d.ts

@@ -13,7 +13,7 @@ export interface RetryStrategy {
 }
 export enum CommitMode {
   Sync = 0,
-  Async = 1,
+  Async = 1
 }
 export interface ConsumerConfiguration {
   groupId: string
@@ -25,7 +25,7 @@ export interface ConsumerConfiguration {
 export enum PartitionPosition {
   Beginning = 'Beginning',
   End = 'End',
-  Stored = 'Stored',
+  Stored = 'Stored'
 }
 export interface OffsetModel {
   offset?: number
@@ -44,7 +44,7 @@ export enum SecurityProtocol {
   Plaintext = 'Plaintext',
   Ssl = 'Ssl',
   SaslPlaintext = 'SaslPlaintext',
-  SaslSsl = 'SaslSsl',
+  SaslSsl = 'SaslSsl'
 }
 export interface KafkaConfiguration {
   brokers: string
@@ -92,8 +92,9 @@ export declare class KafkaConsumer {
   pause(): void
   resume(): void
   unsubscribe(): void
+  shutdownConsumer(): Promise<void>
   seek(topic: string, partition: number, offsetModel: OffsetModel, timeout?: number | undefined | null): void
-  recv(): Promise<Message>
+  recv(): Promise<Message | null>
   commit(topic: string, partition: number, offset: number, commit: CommitMode): void
 }
 export declare class KafkaClientConfig {

package/js-binding.js

@@ -63,7 +63,7 @@ switch (platform) {
     switch (arch) {
       case 'x64':
         localFileExisted = existsSync(
-          join(__dirname, 'kafka-crab-js.win32-x64-msvc.node'),
+          join(__dirname, 'kafka-crab-js.win32-x64-msvc.node')
         )
         try {
           if (localFileExisted) {
@@ -77,7 +77,7 @@ switch (platform) {
         break
       case 'ia32':
         localFileExisted = existsSync(
-          join(__dirname, 'kafka-crab-js.win32-ia32-msvc.node'),
+          join(__dirname, 'kafka-crab-js.win32-ia32-msvc.node')
         )
         try {
           if (localFileExisted) {
@@ -91,7 +91,7 @@ switch (platform) {
         break
       case 'arm64':
         localFileExisted = existsSync(
-          join(__dirname, 'kafka-crab-js.win32-arm64-msvc.node'),
+          join(__dirname, 'kafka-crab-js.win32-arm64-msvc.node')
         )
         try {
           if (localFileExisted) {
@@ -132,7 +132,7 @@ switch (platform) {
         break
       case 'arm64':
         localFileExisted = existsSync(
-          join(__dirname, 'kafka-crab-js.darwin-arm64.node'),
+          join(__dirname, 'kafka-crab-js.darwin-arm64.node')
         )
         try {
           if (localFileExisted) {
@@ -168,7 +168,7 @@ switch (platform) {
       case 'x64':
         if (isMusl()) {
           localFileExisted = existsSync(
-            join(__dirname, 'kafka-crab-js.linux-x64-musl.node'),
+            join(__dirname, 'kafka-crab-js.linux-x64-musl.node')
           )
           try {
             if (localFileExisted) {
@@ -181,7 +181,7 @@ switch (platform) {
           }
         } else {
           localFileExisted = existsSync(
-            join(__dirname, 'kafka-crab-js.linux-x64-gnu.node'),
+            join(__dirname, 'kafka-crab-js.linux-x64-gnu.node')
           )
           try {
             if (localFileExisted) {
@@ -197,7 +197,7 @@ switch (platform) {
       case 'arm64':
         if (isMusl()) {
           localFileExisted = existsSync(
-            join(__dirname, 'kafka-crab-js.linux-arm64-musl.node'),
+            join(__dirname, 'kafka-crab-js.linux-arm64-musl.node')
           )
           try {
             if (localFileExisted) {
@@ -210,7 +210,7 @@ switch (platform) {
           }
         } else {
           localFileExisted = existsSync(
-            join(__dirname, 'kafka-crab-js.linux-arm64-gnu.node'),
+            join(__dirname, 'kafka-crab-js.linux-arm64-gnu.node')
           )
           try {
             if (localFileExisted) {
@@ -226,7 +226,7 @@ switch (platform) {
       case 'arm':
         if (isMusl()) {
           localFileExisted = existsSync(
-            join(__dirname, 'kafka-crab-js.linux-arm-musleabihf.node'),
+            join(__dirname, 'kafka-crab-js.linux-arm-musleabihf.node')
           )
           try {
             if (localFileExisted) {
@@ -239,7 +239,7 @@ switch (platform) {
           }
         } else {
           localFileExisted = existsSync(
-            join(__dirname, 'kafka-crab-js.linux-arm-gnueabihf.node'),
+            join(__dirname, 'kafka-crab-js.linux-arm-gnueabihf.node')
           )
           try {
             if (localFileExisted) {
@@ -255,7 +255,7 @@ switch (platform) {
       case 'riscv64':
         if (isMusl()) {
           localFileExisted = existsSync(
-            join(__dirname, 'kafka-crab-js.linux-riscv64-musl.node'),
+            join(__dirname, 'kafka-crab-js.linux-riscv64-musl.node')
           )
           try {
             if (localFileExisted) {
@@ -268,7 +268,7 @@ switch (platform) {
           }
         } else {
           localFileExisted = existsSync(
-            join(__dirname, 'kafka-crab-js.linux-riscv64-gnu.node'),
+            join(__dirname, 'kafka-crab-js.linux-riscv64-gnu.node')
           )
           try {
             if (localFileExisted) {
@@ -283,7 +283,7 @@ switch (platform) {
         break
       case 's390x':
         localFileExisted = existsSync(
-          join(__dirname, 'kafka-crab-js.linux-s390x-gnu.node'),
+          join(__dirname, 'kafka-crab-js.linux-s390x-gnu.node')
         )
         try {
           if (localFileExisted) {
@@ -310,8 +310,7 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { KafkaConsumer, CommitMode, PartitionPosition, SecurityProtocol, KafkaClientConfig, KafkaProducer } =
-  nativeBinding
+const { KafkaConsumer, CommitMode, PartitionPosition, SecurityProtocol, KafkaClientConfig, KafkaProducer } = nativeBinding
 
 module.exports.KafkaConsumer = KafkaConsumer
 module.exports.CommitMode = CommitMode

package/kafka-client.js

@@ -13,7 +13,7 @@ class KafkaClient {
      */
     constructor(kafkaConfiguration) {
         this.kafkaConfiguration = kafkaConfiguration;
-        this.kafkaClientConfig = new js_binding_js_1.KafkaClientConfig(kafkaConfiguration);
+        this.kafkaClientConfig = new js_binding_js_1.KafkaClientConfig(this.kafkaConfiguration);
     }
     /**
      * Creates a KafkaProducer instance

package/kafka-stream-readable.d.ts

@@ -1,5 +1,6 @@
 import { Readable } from 'stream';
-import { KafkaConsumer, TopicPartitionConfig } from './js-binding';
+import { CommitMode } from './js-binding';
+import { KafkaConsumer, OffsetModel, TopicPartitionConfig } from './js-binding';
 /**
  * KafkaStreamReadable class
  * @extends Readable
@@ -14,6 +15,8 @@ export declare class KafkaStreamReadable extends Readable {
      * Subscribes to topics
      */
     subscribe(topics: string | Array<TopicPartitionConfig>): Promise<void>;
+    seek(topic: string, partition: number, offsetModel: OffsetModel, timeout?: number | undefined): void;
+    commit(topic: string, partition: number, offset: number, commit: CommitMode): void;
     /**
      * Unsubscribe from topics
      */

package/kafka-stream-readable.js

@@ -27,6 +27,12 @@ class KafkaStreamReadable extends stream_1.Readable {
         }
         await this.kafkaConsumer.subscribe(topics);
     }
+    seek(topic, partition, offsetModel, timeout) {
+        this.kafkaConsumer.seek(topic, partition, offsetModel, timeout);
+    }
+    commit(topic, partition, offset, commit) {
+        this.kafkaConsumer.commit(topic, partition, offset, commit);
+    }
     /**
      * Unsubscribe from topics
      */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "1.0.2",
+  "version": "1.0.5",
   "main": "index.js",
   "types": "index.d.ts",
   "module": "commonjs",
@@ -40,13 +40,13 @@
     "url": "https://github.com/inaiat/kafka-crab-js.git"
   },
   "devDependencies": {
-    "@faker-js/faker": "9.2.0",
+    "@faker-js/faker": "9.3.0",
     "@napi-rs/cli": "^2.18.4",
-    "@types/node": "22.9.0",
+    "@types/node": "22.10.1",
     "ava": "6.2.0",
-    "dprint": "^0.47.5",
-    "nanoid": "5.0.8",
-    "typescript": "5.6.3",
+    "dprint": "^0.48.0",
+    "nanoid": "5.0.9",
+    "typescript": "5.7.2",
     "copyfiles": "^2.4.1",
     "@types/copyfiles": "^2.4.4",
     "rimraf": "^6.0.1"
@@ -65,10 +65,10 @@
     "version": "napi version",
     "fmt": "dprint fmt"
   },
-  "packageManager": "pnpm@9.14.2",
+  "packageManager": "pnpm@9.15.2",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "1.0.2",
-    "kafka-crab-js-darwin-x64": "1.0.2",
-    "kafka-crab-js-linux-x64-gnu": "1.0.2"
+    "kafka-crab-js-darwin-arm64": "1.0.5",
+    "kafka-crab-js-darwin-x64": "1.0.5",
+    "kafka-crab-js-linux-x64-gnu": "1.0.5"
   }
 }