kafka-crab-js 0.3.1 → 0.5.4

package/.yarnrc.yml

@@ -1,3 +1,7 @@
+compressionLevel: mixed
+
+enableGlobalCache: false
+
 nodeLinker: node-modules
 
-yarnPath: .yarn/releases/yarn-3.6.0.cjs
+yarnPath: .yarn/releases/yarn-4.1.1.cjs

package/Cargo.toml

@@ -8,20 +8,19 @@ crate-type = ["cdylib"]
 
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2", default-features = false, features = [
+napi = { version = "2.16", default-features = false, features = [
     "napi8",
     "tokio_rt",
     "serde-json",
 ] }
-napi-derive = "2"
+napi-derive = "2.16"
 
 anyhow = { version = "1.0", features = ["backtrace"] }
 
-rdkafka = { version = "0.34", features = [
+rdkafka = { version = "0.36", features = [
     "libz-static",
     "cmake-build",
     "tokio",
-    "ssl",
     "ssl-vendored",
 ] }
 tokio = { version = "1", features = ["full"] }
@@ -34,3 +33,4 @@ napi-build = "2.0.1"
 
 [profile.release]
 lto = true
+strip = "symbols"

package/README.md

@@ -72,5 +72,4 @@ consumer.startConsumer(async (error, {value, partition, offset}) => {
 });
 ```
 ## Examples
-
 You can find examples of using Kafka Crab JS in the [GitHub repository](https://github.com/inaiat/kafka-crab-js).

package/example/sample.mjs

@@ -1,14 +1,17 @@
 /* eslint-disable no-await-in-loop */
 
-import timersPromises from 'node:timers/promises';
+// import timersPromises from 'node:timers/promises';
 import { Buffer } from 'node:buffer';
 import { nanoid } from 'nanoid';
-import { KafkaClient, CommitMode, PartitionPosition, ConsumerResult } from '../index.js';
+import {
+  KafkaClient, CommitMode, PartitionPosition, ConsumerResult,
+} from '../index.js';
 
 const kafkaClient = new KafkaClient({
   brokers: 'localhost:29092',
   clientId: 'my-js-group',
   logLevel: 'info',
+  brokerAddressFamily: 'v4',
 });
 const topic = `topic-${nanoid()}`;
 

package/index.d.ts

@@ -3,19 +3,6 @@
 
 /* auto-generated by NAPI-RS */
 
-export const enum SecurityProtocol {
-  Plaintext = 'Plaintext',
-  Ssl = 'Ssl',
-  SaslPlaintext = 'SaslPlaintext',
-  SaslSsl = 'SaslSsl'
-}
-export interface KafkaConfiguration {
-  brokers: string
-  clientId: string
-  securityProtocol?: SecurityProtocol
-  configuration?: Record<string, string>
-  logLevel?: string
-}
 export const enum CommitMode {
   AutoCommit = 'AutoCommit',
   Sync = 'Sync',
@@ -43,6 +30,20 @@ export interface ConsumerConfiguration {
   enableAutoCommit?: boolean
   configuration?: Record<string, string>
 }
+export const enum SecurityProtocol {
+  Plaintext = 'Plaintext',
+  Ssl = 'Ssl',
+  SaslPlaintext = 'SaslPlaintext',
+  SaslSsl = 'SaslSsl'
+}
+export interface KafkaConfiguration {
+  brokers: string
+  clientId: string
+  securityProtocol?: SecurityProtocol
+  configuration?: Record<string, string>
+  logLevel?: string
+  brokerAddressFamily?: string
+}
 export interface ProducerConfiguration {
   queueTimeout?: number
   thrownOnError?: boolean
@@ -70,9 +71,9 @@ export interface Payload {
   offset: number
 }
 export const enum PartitionPosition {
-  Beginning = 0,
-  End = 1,
-  Stored = 2
+  Beginning = 'Beginning',
+  End = 'End',
+  Stored = 'Stored'
 }
 export interface OffsetModel {
   offset?: number
@@ -93,16 +94,15 @@ export interface MessageModel {
   key?: Buffer
   headers?: Record<string, Buffer>
 }
+export class KafkaConsumer {
+  startConsumer(callback: (error: Error | undefined, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
+}
 export class KafkaClient {
   readonly kafkaConfiguration: KafkaConfiguration
   constructor(kafkaConfiguration: KafkaConfiguration)
   createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
   createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
 }
-export class ProducerHelper { }
-export class KafkaConsumer {
-  startConsumer(callback: (error: Error | undefined, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
-}
 export class KafkaProducer {
   send(producerRecord: ProducerRecord): Promise<Array<RecordMetadata>>
 }

package/index.js

@@ -237,6 +237,49 @@ switch (platform) {
           loadError = e
         }
         break
+      case 'riscv64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-riscv64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-riscv64-musl.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-riscv64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-riscv64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-riscv64-gnu.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-riscv64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 's390x':
+        localFileExisted = existsSync(
+          join(__dirname, 'kafka-crab-js.linux-s390x-gnu.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.linux-s390x-gnu.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-linux-s390x-gnu')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
       default:
         throw new Error(`Unsupported architecture on Linux: ${arch}`)
     }
@@ -252,14 +295,13 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SecurityProtocol, KafkaClient, CommitMode, ConsumerResult, ProducerHelper, KafkaConsumer, KafkaProducer, AutoOffsetReset, PartitionPosition } = nativeBinding
+const { CommitMode, ConsumerResult, KafkaConsumer, SecurityProtocol, KafkaClient, KafkaProducer, AutoOffsetReset, PartitionPosition } = nativeBinding
 
-module.exports.SecurityProtocol = SecurityProtocol
-module.exports.KafkaClient = KafkaClient
 module.exports.CommitMode = CommitMode
 module.exports.ConsumerResult = ConsumerResult
-module.exports.ProducerHelper = ProducerHelper
 module.exports.KafkaConsumer = KafkaConsumer
+module.exports.SecurityProtocol = SecurityProtocol
+module.exports.KafkaClient = KafkaClient
 module.exports.KafkaProducer = KafkaProducer
 module.exports.AutoOffsetReset = AutoOffsetReset
 module.exports.PartitionPosition = PartitionPosition

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.3.1",
+  "version": "0.5.4",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -24,19 +24,20 @@
   ],
   "author": "inaiat@gmail.com",
   "homepage": "https://github.com/inaiat/kafka-crab-js.git",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/inaiat/kafka-crab-js.git"
+  },
   "devDependencies": {
-    "@napi-rs/cli": "^2.16.3",
-    "ava": "^5.3.1",
-    "benny": "^3.7.1",
-    "kafkajs": "^2.2.4",
-    "nanoid": "^4.0.2",
-    "xo": "^0.54.2"
+    "@napi-rs/cli": "^2.18.0",
+    "ava": "^6.1.2",
+    "xo": "^0.58.0"
   },
   "ava": {
     "timeout": "3m"
   },
   "engines": {
-    "node": ">= 10"
+    "node": ">= 18"
   },
   "xo": {
     "space": true,
@@ -54,10 +55,10 @@
     "universal": "napi universal",
     "version": "napi version"
   },
-  "packageManager": "yarn@3.6.0",
+  "packageManager": "yarn@4.1.1",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.3.1",
-    "kafka-crab-js-darwin-x64": "0.3.1",
-    "kafka-crab-js-linux-x64-gnu": "0.3.1"
+    "kafka-crab-js-darwin-arm64": "0.5.4",
+    "kafka-crab-js-darwin-x64": "0.5.4",
+    "kafka-crab-js-linux-x64-gnu": "0.5.4"
   }
 }

package/src/kafka/consumer/consumer_context.rs

@@ -0,0 +1,29 @@
+use rdkafka::consumer::StreamConsumer;
+use tracing::info;
+
+use rdkafka::{
+  client::ClientContext,
+  consumer::{ConsumerContext, Rebalance},
+  error::KafkaResult,
+  topic_partition_list::TopicPartitionList,
+};
+
+pub type LoggingConsumer = StreamConsumer<CustomContext>;
+
+pub struct CustomContext;
+
+impl ClientContext for CustomContext {}
+
+impl ConsumerContext for CustomContext {
+  fn pre_rebalance(&self, rebalance: &Rebalance) {
+    info!("Pre rebalance {:?}", rebalance);
+  }
+
+  fn post_rebalance(&self, rebalance: &Rebalance) {
+    info!("Post rebalance {:?}", rebalance);
+  }
+
+  fn commit_callback(&self, result: KafkaResult<()>, _offsets: &TopicPartitionList) {
+    info!("Committing offsets: {:?}. Offset: {:?}", result, _offsets);
+  }
+}

package/src/kafka/consumer/consumer_thread.rs

@@ -0,0 +1,196 @@
+use napi::{bindgen_prelude::*, threadsafe_function::ThreadsafeFunction};
+
+use rdkafka::{
+  consumer::{stream_consumer::StreamConsumer, CommitMode as RdKfafkaCommitMode, Consumer},
+  message::{BorrowedMessage, Header, OwnedHeaders},
+  producer::FutureRecord,
+  Message,
+};
+
+use std::time::Duration;
+use tracing::{debug, error, info, warn};
+
+use crate::kafka::{
+  consumer::kafka_consumer::RETRY_COUNTER_NAME,
+  kafka_util::{
+    kakfa_headers_to_hashmap, kakfa_headers_to_hashmap_buffer, ExtractValueOnKafkaHashMap,
+  },
+  model::Payload,
+};
+
+use super::kafka_consumer::{ConsumerResult, ProducerHelper};
+
+use super::consumer_context::CustomContext;
+
+pub struct ConsumerThread {
+  pub stream_consumer: StreamConsumer<CustomContext>,
+  pub func: ThreadsafeFunction<Payload>,
+  pub commit_mode: Option<RdKfafkaCommitMode>,
+  pub retries: i32,
+  pub next_topic_on_fail: Option<String>,
+  pub producer: ProducerHelper,
+  pub pause_consumer_duration: Option<Duration>,
+}
+
+impl ConsumerThread {
+  pub async fn start(&self) {
+    loop {
+      if let Some(duration) = &self.pause_consumer_duration {
+        tracing::debug!("Pause consumer for {}", duration.as_millis());
+        tokio::time::sleep(*duration).await;
+      };
+      match self.stream_consumer.recv().await {
+        Err(e) => error!("Error while receiving from stream consumer: {:?}", e),
+        Ok(message) => {
+          let message_result = self.process_message(&message).await;
+          match self.handle_message_result(&message, message_result).await {
+            Ok(_) => {
+              debug!(
+                "Message consumed successfully. Topic: {}, Partition: {}, Offset: {}",
+                message.topic(),
+                message.partition(),
+                message.offset()
+              );
+            }
+            Err(err) => {
+              error!("Message processing failed. Error: {:?}", err);
+            }
+          }
+        }
+      };
+    }
+  }
+
+  async fn process_message(
+    &self,
+    message: &BorrowedMessage<'_>,
+  ) -> anyhow::Result<Option<ConsumerResult>> {
+    match message.payload_view::<[u8]>() {
+      None => Ok(None),
+      Some(Ok(payload)) => {
+        let payload_js = create_payload(message, payload);
+        match self
+          .func
+          .call_async::<Promise<Option<ConsumerResult>>>(Ok(payload_js))
+          .await
+        {
+          Ok(js_result) => match js_result.await {
+            Ok(value) => match value {
+              None | Some(ConsumerResult::Ok) => {
+                if let Some(commit_mode) = self.commit_mode {
+                  match self.stream_consumer.commit_message(message, commit_mode) {
+                    Ok(_) => Ok(Some(ConsumerResult::Ok)),
+                    Err(e) => Err(anyhow::Error::new(e)),
+                  }
+                } else {
+                  Ok(Some(ConsumerResult::Ok))
+                }
+              }
+              Some(ConsumerResult::Retry) => Ok(Some(ConsumerResult::Retry)),
+            },
+            Err(_) => {
+              warn!("Error while calling return function");
+              Ok(Some(ConsumerResult::Ok))
+            }
+          },
+          Err(err) => {
+            warn!("Error while calling function: {:?}", err.to_string());
+            Ok(Some(ConsumerResult::Retry))
+          }
+        }
+      }
+      Some(_) => Err(anyhow::Error::msg(
+        "Error while deserializing message payload",
+      )),
+    }
+  }
+
+  async fn handle_message_result(
+    &self,
+    message: &BorrowedMessage<'_>,
+    message_result: anyhow::Result<Option<ConsumerResult>>,
+  ) -> anyhow::Result<()> {
+    match message_result {
+      Ok(consumer_result) => match consumer_result {
+        Some(result) => match result {
+          ConsumerResult::Ok => Ok(()),
+          ConsumerResult::Retry => match self.send_to_retry_strategy(message).await {
+            Ok(_) => {
+              debug!("Message sent to retry strategy");
+              Ok(())
+            }
+            Err(e) => Err(anyhow::Error::msg(format!(
+              "Error while sending message to retry strategy: {:?}",
+              e
+            ))),
+          },
+        },
+        None => Ok(()),
+      },
+      Err(err) => Err(err),
+    }
+  }
+
+  async fn send_to_retry_strategy(&self, message: &BorrowedMessage<'_>) -> anyhow::Result<()> {
+    let retry_counter: usize = {
+      let headers_map = kakfa_headers_to_hashmap(message.headers());
+      let mut counter = headers_map.get_value(RETRY_COUNTER_NAME).unwrap_or(1);
+      if self.retries > 0 {
+        counter += 1;
+      }
+      counter
+    };
+
+    let next_topic = if self.retries > 0 && retry_counter <= self.retries.try_into().unwrap() {
+      Some(message.topic().to_string())
+    } else {
+      self.next_topic_on_fail.clone()
+    };
+
+    match next_topic {
+      Some(next_topic) => {
+        info!("Message sent to next topic: {}", next_topic);
+        let new_headers = message
+          .headers()
+          .unwrap_or(OwnedHeaders::new().as_borrowed())
+          .detach()
+          .insert(Header {
+            key: RETRY_COUNTER_NAME,
+            value: Some(retry_counter.to_string().as_str()),
+          });
+        let key = message.key().unwrap_or(&[]);
+
+        let record = FutureRecord::to(&next_topic)
+          .key(key)
+          .headers(new_headers)
+          .payload(message.payload().unwrap_or(&[]));
+
+        let _result = self
+          .producer
+          .future_producer
+          .send(record, self.producer.queue_timeout)
+          .await
+          .map_err(|e| anyhow::Error::new(e.0))?;
+        Ok(())
+      }
+      None => {
+        debug!("There is no next topic. Commit message");
+        Ok(())
+      }
+    }
+  }
+}
+
+fn create_payload(message: &BorrowedMessage<'_>, payload: &[u8]) -> Payload {
+  let key: Option<Buffer> = message.key().map(|bytes| bytes.into());
+  let headers = Some(kakfa_headers_to_hashmap_buffer(message.headers()));
+  let payload_js = Payload::new(
+    payload.into(),
+    key,
+    headers,
+    message.topic().to_owned(),
+    message.partition(),
+    message.offset(),
+  );
+  payload_js
+}

package/src/kafka/{kafka_consumer.rs → consumer/kafka_consumer.rs}

@@ -1,58 +1,33 @@
-use napi::{bindgen_prelude::*, threadsafe_function::ThreadsafeFunction, Result};
+use napi::{threadsafe_function::ThreadsafeFunction, Result};
 
 use rdkafka::{
-  client::ClientContext,
   config::RDKafkaLogLevel,
-  consumer::{
-    stream_consumer::StreamConsumer, CommitMode as RdKfafkaCommitMode, Consumer, ConsumerContext,
-    Rebalance,
-  },
-  error::KafkaResult,
-  message::{BorrowedMessage, Header, OwnedHeaders},
-  producer::{FutureProducer, FutureRecord},
+  consumer::{stream_consumer::StreamConsumer, CommitMode as RdKfafkaCommitMode, Consumer},
+  producer::FutureProducer,
   topic_partition_list::TopicPartitionList,
-  ClientConfig, Message,
+  ClientConfig,
 };
 
 use std::{collections::HashMap, time::Duration, vec};
 use tracing::{debug, error, info, warn};
 
-use crate::kafka::kafka_util::{
-  convert_to_rdkafka_offset, kakfa_headers_to_hashmap, ExtractValueOnKafkaHashMap,
-};
-
-use super::{
+use crate::kafka::{
   kafka_admin::KafkaAdmin,
   kafka_client::KafkaClient,
-  kafka_util::{kakfa_headers_to_hashmap_buffer, AnyhowToNapiError},
+  kafka_util::{convert_to_rdkafka_offset, AnyhowToNapiError},
   model::{OffsetModel, PartitionPosition, Payload},
 };
 
-const RETRY_COUNTER_NAME: &str = "kafka-crab-js-retry-counter";
-const DEFAULT_QUEUE_TIMEOUT: u64 = 5000;
-const DEFAULT_PAUSE_DURATION: i64 = 1000;
-const DEFAULT_RETRY_TOPIC_SUFFIX: &str = "-retry";
-const DEFAULT_DLQ_TOPIC_SUFFIX: &str = "-dlq";
-
-type LoggingConsumer = StreamConsumer<CustomContext>;
-
-pub struct CustomContext;
-
-impl ClientContext for CustomContext {}
-
-impl ConsumerContext for CustomContext {
-  fn pre_rebalance(&self, rebalance: &Rebalance) {
-    info!("Pre rebalance {:?}", rebalance);
-  }
-
-  fn post_rebalance(&self, rebalance: &Rebalance) {
-    info!("Post rebalance {:?}", rebalance);
-  }
+use super::{
+  consumer_context::{CustomContext, LoggingConsumer},
+  consumer_thread::ConsumerThread,
+};
 
-  fn commit_callback(&self, result: KafkaResult<()>, _offsets: &TopicPartitionList) {
-    info!("Committing offsets: {:?}. Offset: {:?}", result, _offsets);
-  }
-}
+pub const RETRY_COUNTER_NAME: &str = "kafka-crab-js-retry-counter";
+pub const DEFAULT_QUEUE_TIMEOUT: u64 = 5000;
+pub const DEFAULT_PAUSE_DURATION: i64 = 1000;
+pub const DEFAULT_RETRY_TOPIC_SUFFIX: &str = "-retry";
+pub const DEFAULT_DLQ_TOPIC_SUFFIX: &str = "-dlq";
 
 #[napi(string_enum)]
 #[derive(Debug, PartialEq)]
@@ -81,10 +56,9 @@ pub enum ConsumerResult {
 }
 
 #[derive(Clone)]
-#[napi]
 pub struct ProducerHelper {
-  future_producer: FutureProducer,
-  queue_timeout: Duration,
+  pub future_producer: FutureProducer,
+  pub queue_timeout: Duration,
 }
 
 fn setup_future_producer(
@@ -149,6 +123,11 @@ impl KafkaConsumer {
     })
   }
 
+  // #[napi(ts_args_type = "callback: (error: Error) => Promise<ConsumerResult | undefined>")]
+  // pub async fn create_consumer(&self) -> Result<()> {
+  //   Ok(())
+  // }
+
   #[napi(
     ts_args_type = "callback: (error: Error | undefined, result: Payload) => Promise<ConsumerResult | undefined>"
   )]
@@ -286,29 +265,46 @@ impl KafkaConsumer {
         enable_auto_commit.unwrap_or(true).to_string(),
       )
       .set_log_level(RDKafkaLogLevel::Debug)
-      .create_with_context(context)
-      .expect("Consumer creation failed");
+      .create_with_context(context)?;
 
     if create_topic.unwrap_or(true) {
+      info!("Creating topic: {:?}", topic);
       let admin = KafkaAdmin::new(&self.client_config);
-      admin.create_topic(topic).await?;
+      let result = admin.create_topic(topic).await;
+      if let Err(e) = result {
+        warn!("Fail to create topic {:?}", e);
+        return Err(anyhow::Error::msg(format!("Fail to create topic: {:?}", e)));
+      }
+      info!("Topic created: {:?}", topic)
     }
 
     if let Some(offset) = convert_to_rdkafka_offset(offset) {
       debug!("Setting offset to: {:?}", offset);
-      let metadata = consumer
-        .fetch_metadata(Some(topic), Duration::from_millis(1500))
-        .expect("Fail to retrive metadata from consumer");
+      let metadata = consumer.fetch_metadata(Some(topic), Duration::from_millis(1500))?;
 
       metadata.topics().iter().for_each(|meta_topic| {
         let mut tpl = TopicPartitionList::new();
         meta_topic.partitions().iter().for_each(|meta_partition| {
           tpl.add_partition(topic, meta_partition.id());
         });
-        tpl.set_all_offsets(offset).expect("Fail to set offset");
-        consumer
-          .assign(&tpl)
-          .expect("Assign topic partition list failed");
+        //TODO: Handle error
+        match tpl.set_all_offsets(offset) {
+          Ok(_) => {
+            debug!("Offset set to: {:?}", offset);
+          }
+          Err(e) => {
+            error!("Fail to set offset: {:?}", e)
+          }
+        };
+        //TODO: Handle error
+        match consumer.assign(&tpl) {
+          Ok(_) => {
+            debug!("Assigning topic: {:?}", topic);
+          }
+          Err(e) => {
+            error!("Fail to assign topic: {:?}", e);
+          }
+        }
       });
     }
 
@@ -328,176 +324,3 @@ impl KafkaConsumer {
     Ok(consumer)
   }
 }
-
-pub struct ConsumerThread {
-  stream_consumer: StreamConsumer<CustomContext>,
-  func: ThreadsafeFunction<Payload>,
-  commit_mode: Option<RdKfafkaCommitMode>,
-  retries: i32,
-  next_topic_on_fail: Option<String>,
-  producer: ProducerHelper,
-  pause_consumer_duration: Option<Duration>,
-}
-
-impl ConsumerThread {
-  pub async fn start(&self) {
-    loop {
-      if let Some(duration) = &self.pause_consumer_duration {
-        tracing::debug!("Pause consumer for {}", duration.as_millis());
-        tokio::time::sleep(*duration).await;
-      };
-      match self.stream_consumer.recv().await {
-        Err(e) => error!("Error while receiving from stream consumer: {:?}", e),
-        Ok(message) => {
-          let message_result = self.process_message(&message).await;
-          match self.handle_message_result(&message, message_result).await {
-            Ok(_) => {
-              debug!(
-                "Message consumed successfully. Topic: {}, Partition: {}, Offset: {}",
-                message.topic(),
-                message.partition(),
-                message.offset()
-              );
-            }
-            Err(err) => {
-              error!("Message processing failed. Error: {:?}", err);
-            }
-          }
-        }
-      };
-    }
-  }
-
-  async fn process_message(
-    &self,
-    message: &BorrowedMessage<'_>,
-  ) -> anyhow::Result<Option<ConsumerResult>> {
-    match message.payload_view::<[u8]>() {
-      None => Ok(None),
-      Some(Ok(payload)) => {
-        let payload_js = create_payload(message, payload);
-        match self
-          .func
-          .call_async::<Promise<Option<ConsumerResult>>>(Ok(payload_js))
-          .await
-        {
-          Ok(js_result) => match js_result.await {
-            Ok(value) => match value {
-              None | Some(ConsumerResult::Ok) => {
-                if let Some(commit_mode) = self.commit_mode {
-                  match self.stream_consumer.commit_message(message, commit_mode) {
-                    Ok(_) => Ok(Some(ConsumerResult::Ok)),
-                    Err(e) => Err(anyhow::Error::new(e)),
-                  }
-                } else {
-                  Ok(Some(ConsumerResult::Ok))
-                }
-              }
-              Some(ConsumerResult::Retry) => Ok(Some(ConsumerResult::Retry)),
-            },
-            Err(_) => {
-              warn!("Error while calling return function");
-              Ok(Some(ConsumerResult::Ok))
-            }
-          },
-          Err(err) => {
-            warn!("Error while calling function: {:?}", err.to_string());
-            Ok(Some(ConsumerResult::Retry))
-          }
-        }
-      }
-      Some(_) => Err(anyhow::Error::msg(
-        "Error while deserializing message payload",
-      )),
-    }
-  }
-
-  async fn handle_message_result(
-    &self,
-    message: &BorrowedMessage<'_>,
-    message_result: anyhow::Result<Option<ConsumerResult>>,
-  ) -> anyhow::Result<()> {
-    match message_result {
-      Ok(consumer_result) => match consumer_result {
-        Some(result) => match result {
-          ConsumerResult::Ok => Ok(()),
-          ConsumerResult::Retry => match self.send_to_retry_strategy(message).await {
-            Ok(_) => {
-              debug!("Message sent to retry strategy");
-              Ok(())
-            }
-            Err(e) => Err(anyhow::Error::msg(format!(
-              "Error while sending message to retry strategy: {:?}",
-              e
-            ))),
-          },
-        },
-        None => Ok(()),
-      },
-      Err(err) => Err(err),
-    }
-  }
-
-  async fn send_to_retry_strategy(&self, message: &BorrowedMessage<'_>) -> anyhow::Result<()> {
-    let retry_counter: usize = {
-      let headers_map = kakfa_headers_to_hashmap(message.headers());
-      let mut counter = headers_map.get_value(RETRY_COUNTER_NAME).unwrap_or(1);
-      if self.retries > 0 {
-        counter += 1;
-      }
-      counter
-    };
-
-    let next_topic = if self.retries > 0 && retry_counter <= self.retries.try_into().unwrap() {
-      Some(message.topic().to_string())
-    } else {
-      self.next_topic_on_fail.clone()
-    };
-
-    match next_topic {
-      Some(next_topic) => {
-        info!("Message sent to next topic: {}", next_topic);
-        let new_headers = message
-          .headers()
-          .unwrap_or(OwnedHeaders::new().as_borrowed())
-          .detach()
-          .insert(Header {
-            key: RETRY_COUNTER_NAME,
-            value: Some(retry_counter.to_string().as_str()),
-          });
-        let key = message.key().unwrap_or(&[]);
-
-        let record = FutureRecord::to(&next_topic)
-          .key(key)
-          .headers(new_headers)
-          .payload(message.payload().unwrap_or(&[]));
-
-        let _result = self
-          .producer
-          .future_producer
-          .send(record, self.producer.queue_timeout)
-          .await
-          .map_err(|e| anyhow::Error::new(e.0))?;
-        Ok(())
-      }
-      None => {
-        debug!("There is no next topic. Commit message");
-        Ok(())
-      }
-    }
-  }
-}
-
-fn create_payload(message: &BorrowedMessage<'_>, payload: &[u8]) -> Payload {
-  let key: Option<Buffer> = message.key().map(|bytes| bytes.into());
-  let headers = Some(kakfa_headers_to_hashmap_buffer(message.headers()));
-  let payload_js = Payload::new(
-    payload.into(),
-    key,
-    headers,
-    message.topic().to_owned(),
-    message.partition(),
-    message.offset(),
-  );
-  payload_js
-}

package/src/kafka/consumer/mod.rs

@@ -0,0 +1,3 @@
+pub mod consumer_context;
+pub mod consumer_thread;
+pub mod kafka_consumer;

package/src/kafka/kafka_admin.rs

@@ -33,14 +33,9 @@ impl<'a> KafkaAdmin<'a> {
   }
 
   async fn fetch_config_resource(&self) -> Result<HashMap<String, String>, KafkaError> {
-    let consumer: BaseConsumer = self
-      .client_config
-      .create()
-      .expect("Consumer creation failed");
+    let consumer: BaseConsumer = self.client_config.create()?;
 
-    let metadata = consumer
-      .fetch_metadata(None, Duration::from_secs(5))
-      .expect("Failed to fetch metadata");
+    let metadata = consumer.fetch_metadata(None, Duration::from_secs(5))?;
 
     for broker in metadata.brokers() {
       info!(
@@ -63,9 +58,9 @@ impl<'a> KafkaAdmin<'a> {
   }
 
   pub async fn create_topic(&self, topic_name: &str) -> anyhow::Result<()> {
+    debug!("Fetching broker properties");
     let broker_properties = self.fetch_config_resource().await?.clone();
-
-    // If you want print all properties => dbg!(broker_properties.clone());
+    debug!("Broker properties: {:?}", broker_properties);
 
     self
       .admin_client

package/src/kafka/kafka_client.rs

@@ -12,7 +12,7 @@ use rdkafka::{
 use tracing::{info, Level};
 
 use super::{
-  kafka_consumer::{ConsumerConfiguration, KafkaConsumer},
+  consumer::kafka_consumer::{ConsumerConfiguration, KafkaConsumer},
   kafka_producer::{KafkaProducer, ProducerConfiguration},
 };
 
@@ -62,6 +62,7 @@ pub struct KafkaConfiguration {
   pub security_protocol: Option<SecurityProtocol>,
   pub configuration: Option<HashMap<String, String>>,
   pub log_level: Option<String>,
+  pub broker_address_family: Option<String>,
 }
 
 #[derive(Clone, Debug)]
@@ -103,6 +104,7 @@ impl KafkaClient {
       security_protocol,
       client_id,
       configuration,
+      broker_address_family,
       ..
     } = kafka_configuration.clone();
 
@@ -111,6 +113,10 @@ impl KafkaClient {
     rdkafka_client_config.set_log_level(RDKafkaLogLevel::Debug);
     rdkafka_client_config.set("bootstrap.servers", brokers);
     rdkafka_client_config.set("client.id", client_id);
+    rdkafka_client_config.set(
+      "broker.address.family",
+      broker_address_family.unwrap_or("v4".to_string()),
+    );
     if let Some(security_protocol) = security_protocol {
       rdkafka_client_config.set("security.protocol", security_protocol.to_string());
     }

package/src/kafka/mod.rs

@@ -1,6 +1,6 @@
+pub mod consumer;
 pub mod kafka_admin;
 pub mod kafka_client;
-pub mod kafka_consumer;
 pub mod kafka_producer;
 pub mod kafka_util;
 pub mod model;

package/src/kafka/model.rs

@@ -1,5 +1,3 @@
-use napi::bindgen_prelude::*;
-
 use std::{collections::HashMap, fmt};
 
 use napi::bindgen_prelude::Buffer;
@@ -53,7 +51,7 @@ impl Payload {
   }
 }
 
-#[napi(object)]
+#[napi(string_enum)]
 #[derive(Debug)]
 pub enum PartitionPosition {
   Beginning,