kafka-crab-js 0.0.0 → 0.0.3

package/Cargo.toml

@@ -16,9 +16,6 @@ anyhow = { version= "1.0", features = ["backtrace"]}
 rdkafka = { version = "0.32", features = ["libz-static", "cmake-build", "tokio", "ssl-vendored"] }
 tokio = { version = "1", features = ["full"] }
 
-# log = "0.4"
-# env_logger = "0.10.0"
-
 tracing            = "0.1"
 tracing-subscriber = "0.3"
 

package/README.md

@@ -1,5 +1,16 @@
 Kafka Crab Js
 -------------
 
-Blazingly Fast Kafka Client for NodeJs
+**Blazingly Fast Kafka Client for NodeJs**
 
+Kafka-Crab-JS is a powerful Node.js library that allows developers to interact with Apache Kafka using Rust programming language, seamlessly integrated with Node.js applications. This documentation provides an overview of the Kafka-Crab-JS library, including installation instructions, usage guidelines, and examples.
+
+
+# Installation
+
+To use Kafka-Crab-JS in your Node.js project, follow the steps below:
+
+todo
+
+# Examples
+todo

package/example/sample.mjs

@@ -1,50 +1,69 @@
-import { ConsumerResult, KafkaClient, KafkaCommitMode, PartitionPosition } from '../index.js'
-import timersPromises from 'node:timers/promises'
-import { Buffer } from 'node:buffer';
+/* eslint-disable no-await-in-loop */
+import timersPromises from 'node:timers/promises';
+import {Buffer} from 'node:buffer';
+import {ConsumerResult, KafkaClient, KafkaCommitMode, PartitionPosition} from '../index.js';
 
-const kafkaClient = new KafkaClient("localhost:29092", "my-id")
-const topic = "my-js-topic"
+const kafkaClient = new KafkaClient('localhost:29092', 'my-id');
+const topic = 'my-js-topic';
 
-let counter = 50;
+let counter = 0;
 
 async function process(message) {
-    await timersPromises.setTimeout(1)
-    return "message received: " + message
+  await timersPromises.setTimeout(1);
+  return 'message received: ' + message;
 }
 
 const consumer = kafkaClient.createConsumer({
-    topic, 
-    groupId: "my-js-group",
-    commitMode: KafkaCommitMode.Sync,
-    offset: { position: PartitionPosition.Stored },
-})
-
-console.time("consumer")
-
-consumer.startConsumer(async (err, value)=> {
-    if (err) {
-        console.error("Consumer error", err)
-        return
-    }
-    const message = value.toString()
-    const content = await process(message)
-    counter++
-    if (counter >= 0) {
-        console.timeEnd("consumer")
-        console.log("Counter:", counter, " Content:", content)
-        console.time("consumer")
-    }
-    return ConsumerResult.Ok
-})
-
-await timersPromises.setTimeout(5_000)
-
-const result = await kafkaClient.createProducer().send(
-    topic,
-    {
-        key: Buffer.from("abc"),
-        value: Buffer.from(`{"_id":"${counter}","name":"Elizeu Drummond Sample js","phone":"555"}`),
-        headers: {"key": Buffer.from("value1")}
-    }
-)   
-console.log(result)
+  topic,
+  groupId: 'my-js-group',
+  commitMode: KafkaCommitMode.Sync,
+  offset: {position: PartitionPosition.Stored},
+  retryStrategy: {
+    retries: 3,
+  },
+});
+
+console.time('consumer');
+
+consumer.startConsumer(async (error, value) => {
+  if (error) {
+    console.error('Consumer error', error);
+    return;
+  }
+
+  const message = JSON.parse(value.toString());
+  const content = await process(message);
+  counter++;
+  if (counter >= 0) {
+    console.timeEnd('consumer');
+    console.log('Counter:', counter, 'Content:', JSON.stringify(content));
+    console.time('consumer');
+  }
+
+  if (message._id === '3') {
+    console.error('Error on process message, let\'s retry', message);
+    return ConsumerResult.Retry;
+  }
+
+  return ConsumerResult.Ok;
+});
+
+const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '2'}});
+console.log('Sending message');
+
+for (let i = 0; i < 5; i++) {
+  try {
+    const result = await producer.send(
+      {
+        topic,
+        key: Buffer.from('abc'),
+        value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`),
+        headers: {key: Buffer.from('value1')},
+      },
+    );
+    console.log(result);
+    await timersPromises.setTimeout(1200);
+  } catch (error) {
+    console.error('Error on send', error);
+  }
+}

package/index.d.ts

@@ -9,11 +9,22 @@ export const enum SecurityProtocol {
   SaslPlaintext = 'SaslPlaintext',
   SaslSsl = 'SaslSsl'
 }
-export interface OwnedDelivery {
-  partition: number
-  offset: number
+export interface ConsumerConfiguration {
+  topic: string
+  groupId: string
+  retryStrategy?: RetryStrategy
+  offset?: OffsetModel
+  createTopic?: boolean
+  commitMode?: KafkaCommitMode
+  enableAutoCommit?: boolean
+  configuration?: Record<string, string>
+}
+export interface ProducerConfiguration {
+  queueTimeout?: number
+  configuration?: Record<string, string>
 }
 export interface MessageModel {
+  topic: string
   key: Buffer
   value: Buffer
   headers?: Record<string, Buffer>
@@ -36,13 +47,9 @@ export const enum KafkaCommitMode {
   Sync = 'Sync',
   Async = 'Async'
 }
-export interface ConsumerConfiguration {
-  topic: string
-  groupId: string
-  retryStrategy?: RetryStrategy
-  offset?: OffsetModel
-  createTopic?: boolean
-  commitMode?: KafkaCommitMode
+export interface OwnedDelivery {
+  partition: number
+  offset: number
 }
 export const enum PartitionPosition {
   Beginning = 0,
@@ -55,7 +62,8 @@ export interface OffsetModel {
 }
 export interface RetryStrategy {
   retries: number
-  nextTopicOnFail: string
+  retryTopic?: string
+  dqlTopic?: string
   pauseConsumerDuration?: number
 }
 export const enum ConsumerResult {
@@ -66,17 +74,17 @@ export class KafkaConfiguration {
   brokers: string
   clientId: string
   securityProtocol?: SecurityProtocol
-  allowAutoCreateTopic?: boolean
+  configuration?: Record<string, string>
 }
 export class KafkaClient {
   readonly kafkaConfiguration: KafkaConfiguration
-  constructor(brokers: string, clientId: string)
-  createProducer(): KafkaProducer
+  constructor(brokers: string, clientId: string, securityProtocol?: SecurityProtocol | undefined | null, configuration?: Record<string, string> | undefined | null)
+  createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
   createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
 }
 export class KafkaConsumer {
   startConsumer(callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult>): Promise<void>
 }
 export class KafkaProducer {
-  send(topic: string, message: MessageModel): Promise<OwnedDelivery>
+  send(message: MessageModel): Promise<OwnedDelivery>
 }

package/package.json

@@ -1,22 +1,25 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.0.0",
+  "version": "0.0.3",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
     "name": "kafka-crab-js",
     "triples": {
+      "defaults": false,
       "additional": [
         "aarch64-apple-darwin",
-        "x86_64-unknown-linux-musl"
+        "x86_64-apple-darwin",
+        "x86_64-unknown-linux-gnu"
       ]
     }
   },
   "license": "MIT",
   "devDependencies": {
     "@napi-rs/cli": "^2.16.1",
-    "ava": "^5.1.1",
-    "nanoid": "^4.0.2"
+    "ava": "^5.3.1",
+    "nanoid": "^4.0.2",
+    "xo": "^0.54.2"
   },
   "ava": {
     "timeout": "3m"
@@ -24,6 +27,9 @@
   "engines": {
     "node": ">= 10"
   },
+  "xo": {
+    "space": true
+  },
   "scripts": {
     "artifacts": "napi artifacts",
     "build": "napi build --platform --release",
@@ -35,10 +41,8 @@
   },
   "packageManager": "yarn@3.6.0",
   "optionalDependencies": {
-    "kafka-crab-js-win32-x64-msvc": "0.0.0",
-    "kafka-crab-js-darwin-x64": "0.0.0",
-    "kafka-crab-js-linux-x64-gnu": "0.0.0",
-    "kafka-crab-js-darwin-arm64": "0.0.0",
-    "kafka-crab-js-linux-x64-musl": "0.0.0"
+    "kafka-crab-js-darwin-arm64": "0.0.3",
+    "kafka-crab-js-darwin-x64": "0.0.3",
+    "kafka-crab-js-linux-x64-gnu": "0.0.3"
   }
 }

package/src/kafka/kafka_admin.rs

@@ -65,7 +65,7 @@ impl<'a> KafkaAdmin<'a> {
   pub async fn create_topic(&self, topic_name: &str) -> anyhow::Result<()> {
     let broker_properties = self.fetch_config_resource().await?.clone();
 
-    debug!("Broker properties {:?}", broker_properties);
+    // If you want print all properties => dbg!(broker_properties.clone());
 
     self
       .admin_client

package/src/kafka/kafka_client.rs

@@ -1,6 +1,6 @@
-use std::fmt;
+use std::{collections::HashMap, fmt};
 
-use napi::bindgen_prelude::*;
+use napi::{bindgen_prelude::*, Result};
 use rdkafka::{
   client::ClientContext,
   config::{ClientConfig, RDKafkaLogLevel},
@@ -8,10 +8,12 @@ use rdkafka::{
   error::KafkaResult,
   topic_partition_list::TopicPartitionList,
 };
+
 use tracing::info;
 
 use super::{
-  kafka_consumer::KafkaConsumer, kafka_producer::KafkaProducer, model::ConsumerConfiguration,
+  kafka_consumer::{ConsumerConfiguration, KafkaConsumer},
+  kafka_producer::{KafkaProducer, ProducerConfiguration},
 };
 
 struct CustomContext;
@@ -58,7 +60,7 @@ pub struct KafkaConfiguration {
   pub brokers: String,
   pub client_id: String,
   pub security_protocol: Option<SecurityProtocol>,
-  pub allow_auto_create_topic: Option<bool>,
+  pub configuration: Option<HashMap<String, String>>,
 }
 
 #[derive(Clone, Debug)]
@@ -72,7 +74,12 @@ pub struct KafkaClient {
 #[napi]
 impl KafkaClient {
   #[napi(constructor)]
-  pub fn new(brokers: String, client_id: String) -> Self {
+  pub fn new(
+    brokers: String,
+    client_id: String,
+    security_protocol: Option<SecurityProtocol>,
+    configuration: Option<HashMap<String, String>>,
+  ) -> Self {
     match tracing_subscriber::fmt::try_init() {
       Ok(_) => {}
       Err(e) => println!("Failed to initialize tracing: {:?}", e),
@@ -81,8 +88,8 @@ impl KafkaClient {
     KafkaClient::with_kafka_configuration(KafkaConfiguration {
       brokers,
       client_id,
-      security_protocol: None,
-      allow_auto_create_topic: None,
+      security_protocol,
+      configuration,
     })
   }
 
@@ -95,7 +102,7 @@ impl KafkaClient {
       brokers,
       security_protocol,
       client_id,
-      allow_auto_create_topic,
+      configuration,
     } = kafka_configuration.clone();
 
     let mut rdkafka_client_config = ClientConfig::new();
@@ -106,11 +113,10 @@ impl KafkaClient {
     if let Some(security_protocol) = security_protocol {
       rdkafka_client_config.set("security.protocol", security_protocol.to_string());
     }
-    if let Some(allow_auto_create_topic) = allow_auto_create_topic {
-      rdkafka_client_config.set(
-        "allow.auto.create.topics",
-        allow_auto_create_topic.to_string(),
-      );
+    if let Some(config) = configuration {
+      for (key, value) in config {
+        rdkafka_client_config.set(key, value);
+      }
     }
 
     KafkaClient {
@@ -120,12 +126,21 @@ impl KafkaClient {
   }
 
   #[napi]
-  pub fn create_producer(&self) -> KafkaProducer {
-    KafkaProducer::new(self.rdkafka_client_config.clone())
+  pub fn create_producer(
+    &self,
+    producer_configuration: ProducerConfiguration,
+  ) -> Result<KafkaProducer> {
+    match KafkaProducer::new(self.rdkafka_client_config.clone(), producer_configuration) {
+      Ok(producer) => Ok(producer),
+      Err(e) => Err(Error::new(Status::GenericFailure, e.to_string())),
+    }
   }
 
   #[napi]
-  pub fn create_consumer(&self, consumer_configuration: ConsumerConfiguration) -> KafkaConsumer {
+  pub fn create_consumer(
+    &self,
+    consumer_configuration: ConsumerConfiguration,
+  ) -> Result<KafkaConsumer> {
     KafkaConsumer::new(self.clone(), consumer_configuration)
   }
 }

package/src/kafka/kafka_consumer.rs

@@ -8,20 +8,28 @@ use rdkafka::{
   config::RDKafkaLogLevel,
   consumer::{stream_consumer::StreamConsumer, CommitMode, Consumer, ConsumerContext, Rebalance},
   error::KafkaResult,
-  message::BorrowedMessage,
+  message::{BorrowedMessage, Header, OwnedHeaders},
+  producer::FutureRecord,
   topic_partition_list::TopicPartitionList,
-  Message, Offset,
+  ClientConfig, Message,
 };
 
-use std::{time::Duration, vec};
+use std::{collections::HashMap, time::Duration, vec};
 use tracing::{debug, error, info};
 
+use crate::kafka::kafka_util::{
+  convert_to_rdkafka_offset, kakfa_headers_to_hashmap, ExtractValueOnKafkaHashMap,
+};
+
 use super::{
   kafka_admin::KafkaAdmin,
   kafka_client::KafkaClient,
-  model::{ConsumerConfiguration, ConsumerResult, KafkaCommitMode, OffsetModel, PartitionPosition},
+  kafka_producer::{KafkaProducer, ProducerConfiguration},
+  model::{ConsumerResult, KafkaCommitMode, OffsetModel, RetryStrategy},
 };
 
+const RETRY_COUNTER_NAME: &str = "kafka-crab-js-retry-counter";
+
 type LoggingConsumer = StreamConsumer<CustomContext>;
 
 pub struct CustomContext;
@@ -42,25 +50,56 @@ impl ConsumerContext for CustomContext {
   }
 }
 
+#[napi(object)]
 #[derive(Clone, Debug)]
+pub struct ConsumerConfiguration {
+  pub topic: String,
+  pub group_id: String,
+  pub retry_strategy: Option<RetryStrategy>,
+  pub offset: Option<OffsetModel>,
+  pub create_topic: Option<bool>,
+  pub commit_mode: Option<KafkaCommitMode>,
+  pub enable_auto_commit: Option<bool>,
+  pub configuration: Option<HashMap<String, String>>,
+}
+
+#[derive(Clone)]
 #[napi]
 pub struct KafkaConsumer {
   kafka_client: KafkaClient,
   consumer_configuration: ConsumerConfiguration,
+  producer: KafkaProducer,
 }
 
 #[napi]
 impl KafkaConsumer {
-  pub fn new(kafka_client: KafkaClient, consumer_configuration: ConsumerConfiguration) -> Self {
-    KafkaConsumer {
+  pub fn new(
+    kafka_client: KafkaClient,
+    consumer_configuration: ConsumerConfiguration,
+  ) -> Result<Self> {
+    let producer = KafkaProducer::new(
+      kafka_client.get_client_config().clone(),
+      ProducerConfiguration {
+        configuration: None,
+        queue_timeout: None,
+      },
+    )?;
+
+    Ok(KafkaConsumer {
       kafka_client,
       consumer_configuration,
-    }
+      producer,
+    })
   }
 
   #[napi(ts_args_type = "callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult>")]
   pub async fn start_consumer(&self, func: ThreadsafeFunction<Buffer>) -> Result<()> {
-    let ConsumerConfiguration { commit_mode, .. } = self.consumer_configuration.clone();
+    let ConsumerConfiguration {
+      commit_mode,
+      retry_strategy,
+      topic,
+      ..
+    } = self.consumer_configuration.clone();
 
     let commit_mode = match commit_mode {
       None | Some(KafkaCommitMode::AutoCommit) => None, //Default AutoCommit
@@ -68,40 +107,85 @@ impl KafkaConsumer {
       Some(KafkaCommitMode::Async) => Some(CommitMode::Async),
     };
 
+    let producer = self.producer.clone();
+
+    if let Some(strategy) = retry_strategy.clone() {
+      let retry_topic = strategy
+        .retry_topic
+        .unwrap_or(format!("{}-retry", topic.clone()));
+      let stream_consumer_retry: StreamConsumer<CustomContext> = self
+        .setup_consumer(retry_topic.clone())
+        .await
+        .expect("Consumer creation failed");
+      debug!(
+        "Stream consumer configured for retry topic: {}",
+        retry_topic
+      );
+      let (retry_topic, retry_func, retry_producer) =
+        (topic.clone(), func.clone(), producer.clone());
+      tokio::spawn(async move {
+        start_consumer_loop(
+          stream_consumer_retry,
+          retry_func,
+          commit_mode,
+          strategy.retries,
+          &strategy.dql_topic.unwrap_or(format!("{}-dql", retry_topic)),
+          retry_producer,
+        )
+        .await
+      });
+    };
+
     let stream_consumer: StreamConsumer<CustomContext> = self
-      .setup_consumer()
+      .setup_consumer(topic.clone())
       .await
       .expect("Consumer creation failed");
+    debug!("Stream consumer configured for topic: {}", topic);
+    let next_topic_on_fail = setup_next_topic(
+      self.consumer_configuration.topic.clone(),
+      retry_strategy.clone(),
+    );
 
     tokio::spawn(async move {
-      loop {
-        match stream_consumer.recv().await {
-          Err(e) => error!("Error while receiving from stream consumer: {:?}", e),
-          Ok(message) => process_message(commit_mode, &stream_consumer, &func, &message).await,
-        };
-      }
+      start_consumer_loop(
+        stream_consumer,
+        func,
+        commit_mode,
+        0,
+        &next_topic_on_fail,
+        producer,
+      )
+      .await
     });
 
     Ok(())
   }
 
-  async fn setup_consumer(&self) -> anyhow::Result<StreamConsumer<CustomContext>> {
+  async fn setup_consumer(&self, topic: String) -> anyhow::Result<StreamConsumer<CustomContext>> {
     let context = CustomContext;
 
     let ConsumerConfiguration {
-      topic,
       offset,
       create_topic,
       group_id,
+      configuration,
+      enable_auto_commit,
       ..
     } = self.consumer_configuration.clone();
 
-    let consumer: LoggingConsumer = self
-      .kafka_client
-      .get_client_config()
+    let mut consumer_config: ClientConfig = self.kafka_client.get_client_config().clone();
+
+    if let Some(config) = configuration {
+      consumer_config.extend(config);
+    }
+
+    let consumer: LoggingConsumer = consumer_config
       .clone()
       .set("group.id", group_id.clone())
-      .set("enable.auto.commit", "false")
+      .set(
+        "enable.auto.commit",
+        enable_auto_commit.unwrap_or(true).to_string(),
+      )
       .set_log_level(RDKafkaLogLevel::Debug)
       .create_with_context(context)
       .expect("Consumer creation failed");
@@ -140,49 +224,172 @@ impl KafkaConsumer {
   }
 }
 
-fn convert_to_rdkafka_offset(offset_model: Option<OffsetModel>) -> Option<Offset> {
-  match offset_model {
-    Some(model) => Some(match model.position {
-      Some(PartitionPosition::Beginning) => Offset::Beginning,
-      Some(PartitionPosition::End) => Offset::End,
-      Some(PartitionPosition::Stored) => Offset::Stored,
-      None => match model.offset {
-        Some(value) => Offset::Offset(value),
-        None => Offset::Stored, // Default to stored
+async fn start_consumer_loop(
+  stream_consumer: StreamConsumer<CustomContext>,
+  func: ThreadsafeFunction<Buffer>,
+  commit_mode: Option<CommitMode>,
+  retries: i32,
+  next_topic_on_fail: &str,
+  producer: KafkaProducer,
+) {
+  loop {
+    match stream_consumer.recv().await {
+      Err(e) => error!("Error while receiving from stream consumer: {:?}", e),
+      Ok(message) => {
+        let message_result = process_message(&commit_mode, &stream_consumer, &func, &message).await;
+        match handle_message_result(
+          retries,
+          next_topic_on_fail,
+          &producer,
+          &message,
+          message_result,
+        )
+        .await
+        {
+          Ok(_) => {
+            debug!(
+              "Message consumed successfully. Partition: {}, Offset: {}",
+              message.partition(),
+              message.offset()
+            );
+          }
+          Err(_) => {
+            error!("Message processing failed")
+          }
+        }
+      }
+    };
+  }
+}
+
+async fn handle_message_result(
+  retries: i32,
+  next_topic_on_fail: &str,
+  producer: &KafkaProducer,
+  message: &BorrowedMessage<'_>,
+  message_result: anyhow::Result<Option<ConsumerResult>>,
+) -> anyhow::Result<()> {
+  match message_result {
+    Ok(consumer_result) => match consumer_result {
+      Some(result) => match result {
+        ConsumerResult::Ok => Ok(()),
+        ConsumerResult::Retry => {
+          match send_to_retry_strategy(producer, message, retries, next_topic_on_fail).await {
+            Ok(_) => {
+              debug!("Message sent to retry strategy");
+              Ok(())
+            }
+            Err(e) => {
+              error!("Error while sending message to retry strategy: {:?}", e);
+              Err(e)
+            }
+          }
+        }
       },
-    }),
-    None => None,
+      None => Ok(()),
+    },
+    Err(err) => {
+      error!("Error while processing message: {:?}", err);
+      Err(anyhow::Error::msg("Producer not found"))
+    }
   }
 }
 
 async fn process_message(
-  commit_mode: Option<CommitMode>,
+  commit_mode: &Option<CommitMode>,
   stream_consumer: &StreamConsumer<CustomContext>,
   func: &ThreadsafeFunction<Buffer>,
   message: &BorrowedMessage<'_>,
-) {
+) -> anyhow::Result<Option<ConsumerResult>> {
   match message.payload_view::<[u8]>() {
-    None => {}
+    None => Ok(None),
     Some(Ok(payload)) => match func
       .call_async::<Promise<ConsumerResult>>(Ok(payload.into()))
       .await
     {
       Ok(js_result) => match js_result.await {
-        Ok(value) => {
-          debug!("Message processed: {:?}", value);
-          if let Some(commit_mode) = commit_mode {
-            match stream_consumer.commit_message(&message, commit_mode) {
-              Ok(_) => debug!("Message committed"),
-              Err(e) => error!("Error on commit: {:?}", e),
+        Ok(value) => match value {
+          ConsumerResult::Ok => {
+            if let Some(commit_mode) = commit_mode {
+              match stream_consumer.commit_message(message, *commit_mode) {
+                Ok(_) => Ok(Some(value)),
+                Err(e) => Err(anyhow::Error::new(e)),
+              }
+            } else {
+              Ok(Some(value))
             }
           }
-        }
-        Err(e) => error!("Error on promise: {:?}", e),
+          ConsumerResult::Retry => Ok(Some(value)),
+        },
+        Err(e) => Err(anyhow::Error::new(e)),
       },
-      Err(js_error) => error!("Error on js call: {:?}", js_error),
+      Err(js_error) => Err(anyhow::Error::new(js_error)),
     },
-    Some(Err(e)) => {
-      error!("Error while deserializing message payload: {:?}", e);
+    Some(_) => Err(anyhow::Error::msg(
+      "Error while deserializing message payload",
+    )),
+  }
+}
+
+async fn send_to_retry_strategy(
+  producer: &KafkaProducer,
+  message: &BorrowedMessage<'_>,
+  retries: i32,
+  next_topic_on_fail: &str,
+) -> anyhow::Result<()> {
+  let retry_counter: usize = {
+    let headers_map = kakfa_headers_to_hashmap(message.headers());
+    let mut counter = headers_map.get_value(RETRY_COUNTER_NAME).unwrap_or(1);
+    if retries > 0 {
+      counter += 1;
+    }
+    counter
+  };
+
+  let next_topic = if retries > 0 && retry_counter <= retries.try_into().unwrap() {
+    debug!(
+      "Send message to same topic. Retry counter: {}",
+      retry_counter
+    );
+    message.topic()
+  } else {
+    debug!(
+      "Send message to next topic: {}. Retry counter {}",
+      next_topic_on_fail, retry_counter
+    );
+    next_topic_on_fail
+  };
+  let new_headers = message
+    .headers()
+    .unwrap_or(OwnedHeaders::new().as_borrowed())
+    .detach()
+    .insert(Header {
+      key: RETRY_COUNTER_NAME,
+      value: Some(retry_counter.to_string().as_str()),
+    });
+  let key = message.key().unwrap_or(&[]);
+
+  let record = FutureRecord::to(next_topic)
+    .key(key)
+    .headers(new_headers)
+    .payload(message.payload().unwrap_or(&[]));
+
+  let _result = producer.raw_send(record).await;
+  info!("Message sent to topic: {:?}", next_topic);
+  Ok(())
+}
+
+fn setup_next_topic(main_topic: String, retry_strategy: Option<RetryStrategy>) -> String {
+  match retry_strategy {
+    Some(strategy) => {
+      if (strategy.retries) > 0 {
+        strategy
+          .retry_topic
+          .unwrap_or(format!("{}-retry", main_topic))
+      } else {
+        strategy.dql_topic.unwrap_or(format!("{}-dlq", main_topic))
+      }
     }
+    None => format!("{}-dlq", main_topic),
   }
 }

package/src/kafka/kafka_producer.rs

@@ -1,57 +1,107 @@
-use std::time::Duration;
+use std::{collections::HashMap, time::Duration};
 
 use napi::{Error, Result, Status};
 use rdkafka::{
-  message::ToBytes,
+  message::{OwnedHeaders, ToBytes},
   producer::{FutureProducer, FutureRecord},
   ClientConfig,
 };
+use tracing::{debug, warn};
 
-use super::model::MessageModel;
+use super::{
+  kafka_util::hashmap_to_kafka_headers,
+  model::{MessageModel, OwnedDelivery},
+};
+
+const DEFAULT_QUEUE_TIMEOUT: i64 = 5000;
+
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct ProducerConfiguration {
+  pub queue_timeout: Option<i64>,
+  pub configuration: Option<HashMap<String, String>>,
+}
 
 #[derive(Clone)]
 #[napi]
 pub struct KafkaProducer {
   future_producer: FutureProducer,
-}
-
-#[napi(object)]
-pub struct OwnedDelivery {
-  pub partition: i32,
-  pub offset: i64,
+  queue_timeout: Duration,
 }
 
 #[napi]
 impl KafkaProducer {
-  pub fn new(client_config: ClientConfig) -> Self {
-    let future_producer = client_config
-      .clone()
-      .set("message.timeout.ms", "5000")
+  pub fn new(
+    client_config: ClientConfig,
+    producer_configuration: ProducerConfiguration,
+  ) -> Result<Self> {
+    let mut future_producer_config = client_config;
+
+    if let Some(config) = producer_configuration.configuration {
+      future_producer_config.extend(config);
+    }
+
+    let future_producer = future_producer_config
       .create::<FutureProducer>()
-      .expect("Producer creation error");
+      .map_err(|e| {
+        warn!("Failed to create Kafka producer. Error: {:?}", e);
+        Error::new(Status::GenericFailure, e)
+      })?;
+
+    let queue_timeout = Duration::from_millis(
+      producer_configuration
+        .queue_timeout
+        .unwrap_or(DEFAULT_QUEUE_TIMEOUT)
+        .try_into()
+        .map_err(|e| Error::new(Status::GenericFailure, e))?,
+    );
 
-    KafkaProducer { future_producer }
+    Ok(KafkaProducer {
+      future_producer,
+      queue_timeout,
+    })
   }
 
   #[napi]
-  pub async fn send(&self, topic: String, message: MessageModel) -> Result<OwnedDelivery> {
+  pub async fn send(&self, message: MessageModel) -> Result<OwnedDelivery> {
     let MessageModel {
+      topic,
       key,
       value,
-      headers: _,
+      headers,
     } = message;
 
+    let kafka_headers = match headers {
+      Some(v) => hashmap_to_kafka_headers(&v),
+      None => OwnedHeaders::new(),
+    };
+
     let record = FutureRecord::to(&topic)
       .key(key.to_bytes())
+      .headers(kafka_headers)
       .payload(value.to_bytes());
 
-    match self
-      .future_producer
-      .send(record, Duration::from_secs(5000))
-      .await
-    {
+    match self.raw_send(record).await {
       Ok((partition, offset)) => Ok(OwnedDelivery { partition, offset }),
-      Err((kafka_error, _)) => Err(Error::new(Status::GenericFailure, kafka_error)),
+      Err(e) => Err(Error::new(Status::GenericFailure, e)),
     }
   }
+
+  pub async fn raw_send<'a, K: ToBytes + ?Sized, P: ToBytes + ?Sized>(
+    &self,
+    record: FutureRecord<'a, K, P>,
+  ) -> anyhow::Result<(i32, i64)> {
+    debug!("Trying to produce message. Topic {}", record.topic);
+    let producer: &FutureProducer = &self.future_producer;
+
+    producer
+      .send(record, self.queue_timeout)
+      .await
+      .map_err(|(k, m)| {
+        anyhow::Error::msg(format!(
+          "Failed to send message to Kafka. Error: {:?}. Message: {:?}",
+          k, m
+        ))
+      })
+  }
 }

package/src/kafka/kafka_util.rs

@@ -1,8 +1,36 @@
 use std::collections::HashMap;
 
-use rdkafka::message::{BorrowedHeaders, Headers};
+use napi::bindgen_prelude::Buffer;
+use rdkafka::{
+  message::{BorrowedHeaders, Header, Headers, OwnedHeaders},
+  Offset,
+};
 
-pub fn convert_kakfa_headers_to_hashmap(headers: Option<&BorrowedHeaders>) -> HashMap<&str, &[u8]> {
+use super::model::{OffsetModel, PartitionPosition};
+
+pub fn convert_to_rdkafka_offset(offset_model: Option<OffsetModel>) -> Option<Offset> {
+  offset_model.map(|model| match model.position {
+    Some(PartitionPosition::Beginning) => Offset::Beginning,
+    Some(PartitionPosition::End) => Offset::End,
+    Some(PartitionPosition::Stored) => Offset::Stored,
+    None => match model.offset {
+      Some(value) => Offset::Offset(value),
+      None => Offset::Stored, // Default to stored
+    },
+  })
+}
+
+pub fn hashmap_to_kafka_headers(map: &HashMap<String, Buffer>) -> OwnedHeaders {
+  map.iter().fold(OwnedHeaders::new(), |acc, (key, value)| {
+    let value: &[u8] = value.as_ref();
+    acc.insert(Header {
+      key,
+      value: Some(value),
+    })
+  })
+}
+
+pub fn kakfa_headers_to_hashmap(headers: Option<&BorrowedHeaders>) -> HashMap<&str, &[u8]> {
   match headers {
     Some(value) => value
       .iter()
@@ -39,29 +67,27 @@ impl ExtractValueOnKafkaHashMap<usize> for HashMap<&str, &[u8]> {
 mod tests {
   use std::collections::HashMap;
 
-  use rdkafka::message::{Header, OwnedHeaders};
+  use napi::bindgen_prelude::Buffer;
+  use rdkafka::message::Headers;
 
-  use crate::kafka::kafka_util::{convert_kakfa_headers_to_hashmap, ExtractValueOnKafkaHashMap};
-
-  fn create_header<'a>(key: &'a str, value: Option<&'a str>) -> Header<'a, &'a str> {
-    Header { key, value }
-  }
+  use crate::kafka::kafka_util::{
+    hashmap_to_kafka_headers, kakfa_headers_to_hashmap, ExtractValueOnKafkaHashMap,
+  };
 
   #[test]
-  fn filter_out_nones() {
-    let _x = Header {
-      key: "header_key",
-      value: Some("header_value"),
-    };
-    let headers = OwnedHeaders::new()
-      .insert(create_header("key1", Some("1")))
-      .insert(create_header("key2", None))
-      .insert(create_header("key4", None))
-      .insert(create_header("key5", Some("5")));
-
-    let hash_map = convert_kakfa_headers_to_hashmap(Some(headers.as_borrowed()));
-
-    assert_eq!(hash_map.len(), 2);
+  fn headers_test() {
+    let hash_map: HashMap<String, Buffer> = HashMap::from([
+      ("key_a".to_owned(), "A".as_bytes().into()),
+      ("key_b".to_owned(), "B".as_bytes().into()),
+    ]);
+
+    let rd_headers = hashmap_to_kafka_headers(&hash_map);
+
+    dbg!(rd_headers.get(0));
+    dbg!(rd_headers.get(1));
+
+    let result = kakfa_headers_to_hashmap(Some(rd_headers.as_borrowed()));
+    assert_eq!(result.len(), 2);
   }
 
   #[test]

package/src/kafka/model.rs

@@ -7,6 +7,7 @@ use napi::bindgen_prelude::Buffer;
 #[napi(object)]
 #[derive(Clone)]
 pub struct MessageModel {
+  pub topic: String,
   pub key: Buffer,
   pub value: Buffer,
   pub headers: Option<HashMap<String, Buffer>>,
@@ -38,7 +39,7 @@ impl fmt::Display for AutoOffsetReset {
 }
 
 #[napi(string_enum)]
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub enum KafkaCommitMode {
   AutoCommit,
   Sync,
@@ -46,14 +47,9 @@ pub enum KafkaCommitMode {
 }
 
 #[napi(object)]
-#[derive(Clone, Debug)]
-pub struct ConsumerConfiguration {
-  pub topic: String,
-  pub group_id: String,
-  pub retry_strategy: Option<RetryStrategy>,
-  pub offset: Option<OffsetModel>,
-  pub create_topic: Option<bool>,
-  pub commit_mode: Option<KafkaCommitMode>,
+pub struct OwnedDelivery {
+  pub partition: i32,
+  pub offset: i64,
 }
 
 #[napi(object)]
@@ -74,7 +70,8 @@ pub struct OffsetModel {
 #[derive(Clone, Debug)]
 pub struct RetryStrategy {
   pub retries: i32,
-  pub next_topic_on_fail: String,
+  pub retry_topic: Option<String>,
+  pub dql_topic: Option<String>,
   pub pause_consumer_duration: Option<i64>,
 }