kafka-crab-js 0.6.0 → 0.7.0

package/example/stream_consumer.mjs

@@ -23,7 +23,7 @@ async function produce(topic, messages=1) {
       const result = await producer.send(
         {
           topic,
-          messages: [{ value: Buffer.from(`{"_id":"${i}","name":"${fakerPT_BR.person.fullName()}","phone":"${fakerPT_BR.phone.number()}"}`) }],
+          messages: [{ payload: Buffer.from(`{"_id":"${i}","name":"${fakerPT_BR.person.fullName()}","phone":"${fakerPT_BR.phone.number()}"}`) }],
         },
       );
       console.log('Js message sent. Offset:', result);
@@ -82,11 +82,15 @@ async function startConsumer() {
       counter++;
       console.log('>>> Message received:', { payload: message.payload.toString(), offset: message.offset, partition: message.partition, topic: message.topic })
       if (counter === maxMessages) {
-        kafkaStreamConsumer.unsubscribe();
         kafkaStream.destroy();
       }
       // streamerConsumer.commit(message.partition, message.offset+1, CommitMode.Sync)
     })
+
+    kafkaStream.on('close', () => {
+      kafkaStreamConsumer.unsubscribe();
+      console.log('Stream ended')
+    })
   }
 
   if (process.argv[2] === 'send') {

package/index.d.ts

@@ -27,8 +27,8 @@ export interface RetryStrategy {
   configuration?: Record<string, string>
 }
 export const enum CommitMode {
-  Sync = 'Sync',
-  Async = 'Async'
+  Sync = 0,
+  Async = 1
 }
 export interface ConsumerConfiguration {
   groupId: string
@@ -37,29 +37,6 @@ export interface ConsumerConfiguration {
   configuration?: Record<string, string>
   fecthMetadataTimeout?: number
 }
-export const enum SecurityProtocol {
-  Plaintext = 'Plaintext',
-  Ssl = 'Ssl',
-  SaslPlaintext = 'SaslPlaintext',
-  SaslSsl = 'SaslSsl'
-}
-export interface KafkaConfiguration {
-  brokers: string
-  clientId: string
-  securityProtocol?: SecurityProtocol
-  configuration?: Record<string, string>
-  logLevel?: string
-  brokerAddressFamily?: string
-}
-export const enum AutoOffsetReset {
-  Smallest = 'Smallest',
-  Earliest = 'Earliest',
-  Beginning = 'Beginning',
-  Largest = 'Largest',
-  Latest = 'Latest',
-  End = 'End',
-  Error = 'Error'
-}
 export const enum PartitionPosition {
   Beginning = 'Beginning',
   End = 'End',
@@ -78,26 +55,25 @@ export interface TopicPartitionConfig {
   allOffsets?: OffsetModel
   partitionOffset?: Array<PartitionOffset>
 }
-export interface RecordMetadata {
-  topic: string
-  partition: number
-  offset: number
-  error?: KafkaCrabError
+export const enum SecurityProtocol {
+  Plaintext = 'Plaintext',
+  Ssl = 'Ssl',
+  SaslPlaintext = 'SaslPlaintext',
+  SaslSsl = 'SaslSsl'
 }
-export interface MessageProducer {
-  value: Buffer
-  key?: Buffer
-  headers?: Record<string, Buffer>
+export interface KafkaConfiguration {
+  brokers: string
+  clientId: string
+  securityProtocol?: SecurityProtocol
+  configuration?: Record<string, string>
+  logLevel?: string
+  brokerAddressFamily?: string
 }
 export interface ProducerConfiguration {
   queueTimeout?: number
   thrownOnError?: boolean
   configuration?: Record<string, string>
 }
-export interface ProducerRecord {
-  topic: string
-  messages: Array<MessageProducer>
-}
 export interface Payload {
   value: Buffer
   key?: Buffer
@@ -114,6 +90,21 @@ export interface Message {
   partition: number
   offset: number
 }
+export interface RecordMetadata {
+  topic: string
+  partition: number
+  offset: number
+  error?: KafkaCrabError
+}
+export interface MessageProducer {
+  payload: Buffer
+  key?: Buffer
+  headers?: Record<string, Buffer>
+}
+export interface ProducerRecord {
+  topic: string
+  messages: Array<MessageProducer>
+}
 export interface KafkaCrabError {
   code: number
   message: string

package/index.js

@@ -310,14 +310,13 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { ConsumerResult, KafkaConsumer, KafkaStreamConsumer, CommitMode, SecurityProtocol, KafkaClient, AutoOffsetReset, PartitionPosition, KafkaProducer } = nativeBinding
+const { ConsumerResult, KafkaConsumer, KafkaStreamConsumer, CommitMode, PartitionPosition, SecurityProtocol, KafkaClient, KafkaProducer } = nativeBinding
 
 module.exports.ConsumerResult = ConsumerResult
 module.exports.KafkaConsumer = KafkaConsumer
 module.exports.KafkaStreamConsumer = KafkaStreamConsumer
 module.exports.CommitMode = CommitMode
+module.exports.PartitionPosition = PartitionPosition
 module.exports.SecurityProtocol = SecurityProtocol
 module.exports.KafkaClient = KafkaClient
-module.exports.AutoOffsetReset = AutoOffsetReset
-module.exports.PartitionPosition = PartitionPosition
 module.exports.KafkaProducer = KafkaProducer

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -61,8 +61,8 @@
   },
   "packageManager": "pnpm@9.1.1",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.6.0",
-    "kafka-crab-js-darwin-x64": "0.6.0",
-    "kafka-crab-js-linux-x64-gnu": "0.6.0"
+    "kafka-crab-js-darwin-arm64": "0.7.0",
+    "kafka-crab-js-darwin-x64": "0.7.0",
+    "kafka-crab-js-linux-x64-gnu": "0.7.0"
   }
 }

package/src/kafka/consumer/consumer_helper.rs

@@ -3,18 +3,27 @@ use std::{collections::HashMap, time::Duration};
 use rdkafka::{
   config::RDKafkaLogLevel,
   consumer::{Consumer, StreamConsumer},
-  ClientConfig, TopicPartitionList,
+  ClientConfig, Offset, TopicPartitionList,
 };
 use tracing::{debug, error, info, warn};
 
-use crate::kafka::{
-  consumer::model::LoggingConsumer,
-  kafka_admin::KafkaAdmin,
-  kafka_util::convert_to_rdkafka_offset,
-  model::{OffsetModel, PartitionOffset},
+use crate::kafka::{consumer::model::LoggingConsumer, kafka_admin::KafkaAdmin};
+
+use super::model::{
+  ConsumerConfiguration, CustomContext, OffsetModel, PartitionOffset, PartitionPosition,
 };
 
-use super::{model::ConsumerConfiguration, model::CustomContext};
+pub fn convert_to_rdkafka_offset(offset_model: &OffsetModel) -> Offset {
+  match offset_model.position {
+    Some(PartitionPosition::Beginning) => Offset::Beginning,
+    Some(PartitionPosition::End) => Offset::End,
+    Some(PartitionPosition::Stored) => Offset::Stored,
+    None => match offset_model.offset {
+      Some(value) => Offset::Offset(value),
+      None => Offset::Stored, // Default to stored
+    },
+  }
+}
 
 pub async fn create_stream_consumer_and_setup_everything(
   client_config: &ClientConfig,
@@ -27,14 +36,14 @@ pub async fn create_stream_consumer_and_setup_everything(
   let consumer = create_stream_consumer(client_config, consumer_configuration, configuration)?;
 
   if consumer_configuration.create_topic.unwrap_or(true) {
-    try_create_topic(topic, client_config).await?;
+    try_create_topic(&vec![topic.to_owned()], client_config, timeout).await?;
   }
 
   if let Some(offset_model) = offset {
     set_offset_of_all_partitions(offset_model, &consumer, topic, timeout)?;
   }
 
-  try_subscribe(&consumer, topic)?;
+  try_subscribe(&consumer, &vec![topic.to_owned()])?;
 
   Ok(consumer)
 }
@@ -77,28 +86,30 @@ pub fn create_stream_consumer(
   Ok(consumer)
 }
 
-pub fn try_subscribe(consumer: &LoggingConsumer, topic: &str) -> anyhow::Result<()> {
-  consumer
-    .subscribe(vec![&*topic.to_string()].as_slice())
-    .map_err(|e| {
-      anyhow::Error::msg(format!(
-        "Can't subscribe to specified topics. Error: {:?}",
-        e
-      ))
-    })?;
-  debug!("Subscribed to topic: {:?}", topic);
+pub fn try_subscribe(consumer: &LoggingConsumer, topics: &Vec<String>) -> anyhow::Result<()> {
+  let topics_ref = topics.iter().map(|x| x.as_str()).collect::<Vec<&str>>();
+  consumer.subscribe(topics_ref.as_slice()).map_err(|e| {
+    anyhow::Error::msg(format!(
+      "Can't subscribe to specified topic(s): {:?}. Error: {:?}",
+      topics_ref, e
+    ))
+  })?;
+  debug!("Subscribed to topic(s): {:?}", topics_ref);
   Ok(())
 }
 
-pub async fn try_create_topic(topic: &str, client_config: &ClientConfig) -> anyhow::Result<()> {
-  info!("Creating topic: {:?}", topic);
-  let admin = KafkaAdmin::new(client_config, None);
-  let result = admin.create_topic(topic).await;
+pub async fn try_create_topic(
+  topics: &Vec<String>,
+  client_config: &ClientConfig,
+  fetch_metadata_timeout: Duration,
+) -> anyhow::Result<()> {
+  let admin = KafkaAdmin::new(client_config, Some(fetch_metadata_timeout));
+  let result = admin.create_topic(topics).await;
   if let Err(e) = result {
     warn!("Fail to create topic {:?}", e);
     return Err(anyhow::Error::msg(format!("Fail to create topic: {:?}", e)));
   }
-  info!("Topic created: {:?}", topic);
+  info!("Topic(s) created: {:?}", topics);
   Ok(())
 }
 

package/src/kafka/consumer/consumer_thread.rs

@@ -12,7 +12,9 @@ use tracing::{debug, error, info, warn};
 
 use crate::kafka::{
   consumer::kafka_consumer::RETRY_COUNTER_NAME,
-  kafka_util::{create_payload, kakfa_headers_to_hashmap, ExtractValueOnKafkaHashMap},
+  kafka_util::{
+    kakfa_headers_to_hashmap, kakfa_headers_to_hashmap_buffer, ExtractValueOnKafkaHashMap,
+  },
   producer::model::Payload,
 };
 
@@ -179,3 +181,17 @@ impl ConsumerThread {
     }
   }
 }
+
+pub fn create_payload(message: &BorrowedMessage<'_>, payload: &[u8]) -> Payload {
+  let key: Option<Buffer> = message.key().map(|bytes| bytes.into());
+  let headers = Some(kakfa_headers_to_hashmap_buffer(message.headers()));
+  let payload_js = Payload::new(
+    payload.into(),
+    key,
+    headers,
+    message.topic().to_owned(),
+    message.partition(),
+    message.offset(),
+  );
+  payload_js
+}

package/src/kafka/consumer/kafka_consumer.rs

@@ -5,16 +5,16 @@ use rdkafka::{consumer::CommitMode as RdKfafkaCommitMode, producer::FutureProduc
 use std::{collections::HashMap, time::Duration};
 
 use crate::kafka::{
-  kafka_client::KafkaClient,
-  kafka_util::AnyhowToNapiError,
-  model::{OffsetModel, PartitionPosition},
-  producer::model::Payload,
+  kafka_client::KafkaClient, kafka_util::AnyhowToNapiError, producer::model::Payload,
 };
 
 use super::{
   consumer_helper::create_stream_consumer_and_setup_everything,
   consumer_thread::ConsumerThread,
-  model::{CommitMode, ConsumerConfiguration, RetryStrategy, DEFAULT_FECTH_METADATA_TIMEOUT},
+  model::{
+    CommitMode, ConsumerConfiguration, OffsetModel, PartitionPosition, RetryStrategy,
+    DEFAULT_FECTH_METADATA_TIMEOUT,
+  },
 };
 
 pub const RETRY_COUNTER_NAME: &str = "kafka-crab-js-retry-counter";
@@ -103,7 +103,7 @@ impl KafkaConsumer {
       commit_mode,
       topic,
       fetch_metadata_timeout: Duration::from_millis(
-        fecth_metadata_timeout.unwrap_or(DEFAULT_FECTH_METADATA_TIMEOUT) as u64,
+        fecth_metadata_timeout.unwrap_or(DEFAULT_FECTH_METADATA_TIMEOUT.as_millis() as i64) as u64,
       ),
     })
   }
@@ -229,6 +229,6 @@ fn convert_to_consumer_configuration(config: &KafkaConsumerConfiguration) -> Con
     create_topic: config.create_topic,
     enable_auto_commit: config.enable_auto_commit,
     configuration: config.configuration.clone(),
-    fecth_metadata_timeout: Some(DEFAULT_FECTH_METADATA_TIMEOUT),
+    fecth_metadata_timeout: config.fecth_metadata_timeout,
   }
 }

package/src/kafka/consumer/kafka_stream_consumer.rs

@@ -11,16 +11,20 @@ use rdkafka::{
 use tracing::{debug, error, info};
 
 use crate::kafka::{
-  consumer::consumer_helper::{assign_offset_or_use_metadata, try_create_topic},
+  consumer::consumer_helper::{
+    assign_offset_or_use_metadata, convert_to_rdkafka_offset, try_create_topic, try_subscribe,
+  },
   kafka_client::KafkaClient,
-  kafka_util::{convert_to_rdkafka_offset, create_message, AnyhowToNapiError},
-  model::{OffsetModel, TopicPartitionConfig},
+  kafka_util::{create_message, AnyhowToNapiError},
   producer::model::Message,
 };
 
 use super::{
   consumer_helper::{create_stream_consumer, set_offset_of_all_partitions},
-  model::{CommitMode, ConsumerConfiguration, CustomContext, DEFAULT_FECTH_METADATA_TIMEOUT},
+  model::{
+    CommitMode, ConsumerConfiguration, CustomContext, OffsetModel, TopicPartitionConfig,
+    DEFAULT_FECTH_METADATA_TIMEOUT,
+  },
 };
 
 pub const DEFAULT_SEEK_TIMEOUT: i64 = 1500;
@@ -51,7 +55,7 @@ impl KafkaStreamConsumer {
       fecth_metadata_timeout: Duration::from_millis(
         consumer_configuration
           .fecth_metadata_timeout
-          .unwrap_or(DEFAULT_FECTH_METADATA_TIMEOUT) as u64,
+          .unwrap_or(DEFAULT_FECTH_METADATA_TIMEOUT.as_millis() as i64) as u64,
       ),
     })
   }
@@ -63,7 +67,7 @@ impl KafkaStreamConsumer {
   ) -> Result<()> {
     let topics = match topic_configs {
       Either::A(config) => {
-        info!("1.Subscribing to topic: {:#?}", &config);
+        debug!("Subscribing to topic: {:#?}", &config);
         vec![TopicPartitionConfig {
           topic: config,
           all_offsets: None,
@@ -71,7 +75,7 @@ impl KafkaStreamConsumer {
         }]
       }
       Either::B(config) => {
-        info!("2.Subscribing to topics: {:#?}", &config);
+        debug!("Subscribing to topic config: {:#?}", &config);
         config
       }
     };
@@ -80,28 +84,17 @@ impl KafkaStreamConsumer {
       .iter()
       .map(|x| x.topic.clone())
       .collect::<Vec<String>>();
-    let topics_ref = topics_name
-      .iter()
-      .map(|x| x.as_str())
-      .collect::<Vec<&str>>();
 
-    info!("Creating topics if not exists: {:?}", &topics_ref);
-    for topic in &topics_ref {
-      try_create_topic(&topic, &self.client_config)
-        .await
-        .map_err(|e| e.convert_to_napi())?;
-    }
+    debug!("Creating topics if not exists: {:?}", &topics_name);
+    try_create_topic(
+      &topics_name,
+      &self.client_config,
+      self.fecth_metadata_timeout,
+    )
+    .await
+    .map_err(|e| e.convert_to_napi())?;
 
-    info!("Subscribing to topics: {:?}", topics_name);
-    self
-      .stream_consumer
-      .subscribe(&topics_ref.as_slice())
-      .map_err(|e| {
-        napi::Error::new(
-          napi::Status::GenericFailure,
-          format!("Error while subscribing to topic: {:?}", e),
-        )
-      })?;
+    try_subscribe(&self.stream_consumer, &topics_name).map_err(|e| e.convert_to_napi())?;
 
     topics.iter().for_each(|item| {
       if let Some(all_offsets) = item.all_offsets.clone() {

package/src/kafka/consumer/model.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, time::Duration};
 
 use rdkafka::{
   consumer::{ConsumerContext, Rebalance, StreamConsumer},
@@ -7,12 +7,10 @@ use rdkafka::{
 };
 use tracing::info;
 
-use crate::kafka::model::OffsetModel;
+pub const DEFAULT_FECTH_METADATA_TIMEOUT: Duration = Duration::from_millis(2000);
 
 pub type LoggingConsumer = StreamConsumer<CustomContext>;
 
-pub const DEFAULT_FECTH_METADATA_TIMEOUT: i64 = 2000;
-
 pub struct CustomContext;
 
 impl ClientContext for CustomContext {}
@@ -42,11 +40,11 @@ pub struct RetryStrategy {
   pub configuration: Option<HashMap<String, String>>,
 }
 
-#[napi(string_enum)]
+#[napi]
 #[derive(Debug, PartialEq)]
 pub enum CommitMode {
-  Sync,
-  Async,
+  Sync = 0,
+  Async = 1,
 }
 
 #[napi(object)]
@@ -58,3 +56,32 @@ pub struct ConsumerConfiguration {
   pub configuration: Option<HashMap<String, String>>,
   pub fecth_metadata_timeout: Option<i64>,
 }
+
+#[napi(string_enum)]
+#[derive(Debug)]
+pub enum PartitionPosition {
+  Beginning,
+  End,
+  Stored,
+}
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct OffsetModel {
+  pub offset: Option<i64>,
+  pub position: Option<PartitionPosition>,
+}
+
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct PartitionOffset {
+  pub partition: i32,
+  pub offset: OffsetModel,
+}
+
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct TopicPartitionConfig {
+  pub topic: String,
+  pub all_offsets: Option<OffsetModel>,
+  pub partition_offset: Option<Vec<PartitionOffset>>,
+}

package/src/kafka/kafka_admin.rs

@@ -12,7 +12,7 @@ use tracing::{debug, info, trace};
 
 use std::{collections::HashMap, str::FromStr, time::Duration};
 
-use super::consumer::model::DEFAULT_FECTH_METADATA_TIMEOUT;
+const DEFAULT_FECTH_METADATA_TIMEOUT: Duration = Duration::from_millis(2000);
 
 const DEFAULT_NUM_PARTITIONS: i32 = 3;
 const DEFAULT_REPLICATION: i32 = 3;
@@ -24,7 +24,7 @@ pub struct KafkaAdmin<'a> {
 }
 
 impl<'a> KafkaAdmin<'a> {
-  pub fn new(client_config: &'a ClientConfig, fetch_metadata_timeout: Option<i64>) -> Self {
+  pub fn new(client_config: &'a ClientConfig, fetch_metadata_timeout: Option<Duration>) -> Self {
     let admin_client: AdminClient<DefaultClientContext> = client_config
       .create()
       .expect("admin client creation failed");
@@ -32,9 +32,7 @@ impl<'a> KafkaAdmin<'a> {
     KafkaAdmin {
       client_config,
       admin_client,
-      fetch_metadata_timeout: Duration::from_millis(
-        fetch_metadata_timeout.unwrap_or(DEFAULT_FECTH_METADATA_TIMEOUT) as u64,
-      ),
+      fetch_metadata_timeout: fetch_metadata_timeout.unwrap_or(DEFAULT_FECTH_METADATA_TIMEOUT),
     }
   }
 
@@ -63,31 +61,33 @@ impl<'a> KafkaAdmin<'a> {
     Ok(extract_config_resource(config_resource))
   }
 
-  pub async fn create_topic(&self, topic_name: &str) -> anyhow::Result<()> {
+  pub async fn create_topic(&self, topics: &Vec<String>) -> anyhow::Result<()> {
     let broker_properties = self.fetch_config_resource().await?.clone();
     trace!("Broker properties: {:?}", broker_properties);
 
+    let new_topics: Vec<NewTopic> = topics
+      .iter()
+      .map(|topic| NewTopic {
+        name: topic,
+        num_partitions: broker_properties
+          .get("num.partitions")
+          .get_parsed_or_default_value(DEFAULT_NUM_PARTITIONS),
+        replication: TopicReplication::Fixed(
+          broker_properties
+            .get("default.replication.factor")
+            .get_parsed_or_default_value(DEFAULT_REPLICATION),
+        ),
+        config: vec![],
+      })
+      .collect();
+
     self
       .admin_client
-      .create_topics(
-        &[NewTopic {
-          name: topic_name,
-          num_partitions: broker_properties
-            .get("num.partitions")
-            .get_parsed_or_default_value(DEFAULT_NUM_PARTITIONS),
-          replication: TopicReplication::Fixed(
-            broker_properties
-              .get("default.replication.factor")
-              .get_parsed_or_default_value(DEFAULT_REPLICATION),
-          ),
-          config: vec![],
-        }],
-        &AdminOptions::default(),
-      )
+      .create_topics(&new_topics, &AdminOptions::default())
       .await
       .map_err(anyhow::Error::new)?;
 
-    info!("Topic {} was created successfully", topic_name);
+    info!("Topic(s) {:?} was created successfully", topics);
     Ok(())
   }
 }

package/src/kafka/kafka_client.rs

@@ -1,15 +1,9 @@
 use std::{collections::HashMap, fmt, str::FromStr};
 
 use napi::{bindgen_prelude::*, Result};
-use rdkafka::{
-  client::ClientContext,
-  config::{ClientConfig, RDKafkaLogLevel},
-  consumer::{ConsumerContext, Rebalance},
-  error::KafkaResult,
-  topic_partition_list::TopicPartitionList,
-};
+use rdkafka::config::{ClientConfig, RDKafkaLogLevel};
 
-use tracing::{info, Level};
+use tracing::Level;
 
 use super::{
   consumer::{
@@ -20,24 +14,6 @@ use super::{
   producer::kafka_producer::{KafkaProducer, ProducerConfiguration},
 };
 
-struct CustomContext;
-
-impl ClientContext for CustomContext {}
-
-impl ConsumerContext for CustomContext {
-  fn pre_rebalance(&self, rebalance: &Rebalance) {
-    info!("Pre rebalance {:?}", rebalance);
-  }
-
-  fn post_rebalance(&self, rebalance: &Rebalance) {
-    info!("Post rebalance {:?}", rebalance);
-  }
-
-  fn commit_callback(&self, result: KafkaResult<()>, _offsets: &TopicPartitionList) {
-    info!("Committing offsets: {:?}. Offset: {:?}", result, _offsets);
-  }
-}
-
 #[derive(Debug)]
 #[napi(string_enum)]
 pub enum SecurityProtocol {

package/src/kafka/kafka_util.rs

@@ -3,13 +3,10 @@ use std::collections::HashMap;
 use napi::{bindgen_prelude::Buffer, Status};
 use rdkafka::{
   message::{BorrowedHeaders, BorrowedMessage, Header, Headers, OwnedHeaders},
-  Message as RdMessage, Offset,
+  Message as RdMessage,
 };
 
-use super::{
-  model::{OffsetModel, PartitionPosition},
-  producer::model::{Message, Payload},
-};
+use super::producer::model::Message;
 
 pub trait AnyhowToNapiError {
   fn convert_to_napi(&self) -> napi::Error;
@@ -21,18 +18,6 @@ impl AnyhowToNapiError for anyhow::Error {
   }
 }
 
-pub fn convert_to_rdkafka_offset(offset_model: &OffsetModel) -> Offset {
-  match offset_model.position {
-    Some(PartitionPosition::Beginning) => Offset::Beginning,
-    Some(PartitionPosition::End) => Offset::End,
-    Some(PartitionPosition::Stored) => Offset::Stored,
-    None => match offset_model.offset {
-      Some(value) => Offset::Offset(value),
-      None => Offset::Stored, // Default to stored
-    },
-  }
-}
-
 pub fn hashmap_to_kafka_headers(map: &HashMap<String, Buffer>) -> OwnedHeaders {
   map.iter().fold(OwnedHeaders::new(), |acc, (key, value)| {
     let value: &[u8] = value.as_ref();
@@ -89,20 +74,6 @@ impl ExtractValueOnKafkaHashMap<usize> for HashMap<&str, &[u8]> {
   }
 }
 
-pub fn create_payload(message: &BorrowedMessage<'_>, payload: &[u8]) -> Payload {
-  let key: Option<Buffer> = message.key().map(|bytes| bytes.into());
-  let headers = Some(kakfa_headers_to_hashmap_buffer(message.headers()));
-  let payload_js = Payload::new(
-    payload.into(),
-    key,
-    headers,
-    message.topic().to_owned(),
-    message.partition(),
-    message.offset(),
-  );
-  payload_js
-}
-
 pub fn create_message(message: &BorrowedMessage<'_>, payload: &[u8]) -> Message {
   let key: Option<Buffer> = message.key().map(|bytes| bytes.into());
   let headers = Some(kakfa_headers_to_hashmap_buffer(message.headers()));

package/src/kafka/mod.rs

@@ -2,5 +2,4 @@ pub mod consumer;
 pub mod kafka_admin;
 pub mod kafka_client;
 pub mod kafka_util;
-pub mod model;
 pub mod producer;

package/src/kafka/producer/kafka_producer.rs

@@ -16,11 +16,9 @@ use rdkafka::{
   ClientConfig, ClientContext, Message, Statistics,
 };
 
-use napi::bindgen_prelude::Buffer;
-
 use crate::kafka::kafka_util::hashmap_to_kafka_headers;
 
-use super::model::KafkaCrabError;
+use super::model::{KafkaCrabError, MessageProducer, ProducerRecord, RecordMetadata};
 
 const DEFAULT_QUEUE_TIMEOUT: i64 = 5000;
 
@@ -82,23 +80,6 @@ where
     .unwrap()
 }
 
-#[napi(object)]
-#[derive(Clone)]
-pub struct RecordMetadata {
-  pub topic: String,
-  pub partition: i32,
-  pub offset: i64,
-  pub error: Option<KafkaCrabError>,
-}
-
-#[napi(object)]
-#[derive(Clone)]
-pub struct MessageProducer {
-  pub value: Buffer,
-  pub key: Option<Buffer>,
-  pub headers: Option<HashMap<String, Buffer>>,
-}
-
 #[napi(object)]
 #[derive(Clone, Debug)]
 pub struct ProducerConfiguration {
@@ -107,13 +88,6 @@ pub struct ProducerConfiguration {
   pub configuration: Option<HashMap<String, String>>,
 }
 
-#[napi(object)]
-#[derive(Clone)]
-pub struct ProducerRecord {
-  pub topic: String,
-  pub messages: Vec<MessageProducer>,
-}
-
 #[napi]
 pub struct KafkaProducer {
   queue_timeout: Duration,
@@ -153,7 +127,9 @@ impl KafkaProducer {
     let producer = threaded_producer_with_context(context.clone(), self.client_config.clone());
 
     for message in producer_record.messages {
-      let MessageProducer { value, headers, .. } = message;
+      let MessageProducer {
+        payload, headers, ..
+      } = message;
       let headers = match headers {
         Some(v) => hashmap_to_kafka_headers(&v),
         None => OwnedHeaders::new(),
@@ -167,7 +143,7 @@ impl KafkaProducer {
       producer
         .send(
           BaseRecord::to(topic)
-            .payload(value.to_bytes())
+            .payload(payload.to_bytes())
             .headers(headers)
             .key(rd_key),
         )

package/src/kafka/producer/model.rs

@@ -65,6 +65,30 @@ impl Message {
   }
 }
 
+#[napi(object)]
+#[derive(Clone)]
+pub struct RecordMetadata {
+  pub topic: String,
+  pub partition: i32,
+  pub offset: i64,
+  pub error: Option<KafkaCrabError>,
+}
+
+#[napi(object)]
+#[derive(Clone)]
+pub struct MessageProducer {
+  pub payload: Buffer,
+  pub key: Option<Buffer>,
+  pub headers: Option<HashMap<String, Buffer>>,
+}
+
+#[napi(object)]
+#[derive(Clone)]
+pub struct ProducerRecord {
+  pub topic: String,
+  pub messages: Vec<MessageProducer>,
+}
+
 #[napi(object)]
 #[derive(Clone)]
 pub struct KafkaCrabError {

package/src/kafka/model.rs

@@ -1,48 +0,0 @@
-use std::fmt;
-
-#[derive(Debug)]
-#[napi(string_enum)]
-pub enum AutoOffsetReset {
-  Smallest,
-  Earliest,
-  Beginning,
-  Largest,
-  Latest,
-  End,
-  Error,
-}
-
-impl fmt::Display for AutoOffsetReset {
-  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-    write!(f, "{}", format!("{:?}", self).to_lowercase())
-  }
-}
-
-#[napi(string_enum)]
-#[derive(Debug)]
-pub enum PartitionPosition {
-  Beginning,
-  End,
-  Stored,
-}
-#[napi(object)]
-#[derive(Clone, Debug)]
-pub struct OffsetModel {
-  pub offset: Option<i64>,
-  pub position: Option<PartitionPosition>,
-}
-
-#[napi(object)]
-#[derive(Clone, Debug)]
-pub struct PartitionOffset {
-  pub partition: i32,
-  pub offset: OffsetModel,
-}
-
-#[napi(object)]
-#[derive(Clone, Debug)]
-pub struct TopicPartitionConfig {
-  pub topic: String,
-  pub all_offsets: Option<OffsetModel>,
-  pub partition_offset: Option<Vec<PartitionOffset>>,
-}