kafka-crab-js 0.0.0

package/.yarnrc.yml

@@ -0,0 +1,3 @@
+nodeLinker: node-modules
+
+yarnPath: .yarn/releases/yarn-3.6.0.cjs

package/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+edition = "2021"
+name = "kafka-crab-js"
+version = "0.0.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
+napi = { version = "2.13", default-features = false, features = ["napi4", "tokio_rt"] }
+napi-derive = "2.13"
+
+anyhow = { version= "1.0", features = ["backtrace"]}
+
+rdkafka = { version = "0.32", features = ["libz-static", "cmake-build", "tokio", "ssl-vendored"] }
+tokio = { version = "1", features = ["full"] }
+
+# log = "0.4"
+# env_logger = "0.10.0"
+
+tracing            = "0.1"
+tracing-subscriber = "0.3"
+
+[build-dependencies]
+napi-build = "2.0.1"
+
+[profile.release]
+lto = true

package/README.md

@@ -0,0 +1,5 @@
+Kafka Crab Js
+-------------
+
+Blazingly Fast Kafka Client for NodeJs
+

package/build.rs

@@ -0,0 +1,5 @@
+extern crate napi_build;
+
+fn main() {
+  napi_build::setup();
+}

package/example/sample.mjs

@@ -0,0 +1,50 @@
+import { ConsumerResult, KafkaClient, KafkaCommitMode, PartitionPosition } from '../index.js'
+import timersPromises from 'node:timers/promises'
+import { Buffer } from 'node:buffer';
+
+const kafkaClient = new KafkaClient("localhost:29092", "my-id")
+const topic = "my-js-topic"
+
+let counter = 50;
+
+async function process(message) {
+    await timersPromises.setTimeout(1)
+    return "message received: " + message
+}
+
+const consumer = kafkaClient.createConsumer({
+    topic, 
+    groupId: "my-js-group",
+    commitMode: KafkaCommitMode.Sync,
+    offset: { position: PartitionPosition.Stored },
+})
+
+console.time("consumer")
+
+consumer.startConsumer(async (err, value)=> {
+    if (err) {
+        console.error("Consumer error", err)
+        return
+    }
+    const message = value.toString()
+    const content = await process(message)
+    counter++
+    if (counter >= 0) {
+        console.timeEnd("consumer")
+        console.log("Counter:", counter, " Content:", content)
+        console.time("consumer")
+    }
+    return ConsumerResult.Ok
+})
+
+await timersPromises.setTimeout(5_000)
+
+const result = await kafkaClient.createProducer().send(
+    topic,
+    {
+        key: Buffer.from("abc"),
+        value: Buffer.from(`{"_id":"${counter}","name":"Elizeu Drummond Sample js","phone":"555"}`),
+        headers: {"key": Buffer.from("value1")}
+    }
+)   
+console.log(result)

package/index.d.ts

@@ -0,0 +1,82 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+export const enum SecurityProtocol {
+  Plaintext = 'Plaintext',
+  Ssl = 'Ssl',
+  SaslPlaintext = 'SaslPlaintext',
+  SaslSsl = 'SaslSsl'
+}
+export interface OwnedDelivery {
+  partition: number
+  offset: number
+}
+export interface MessageModel {
+  key: Buffer
+  value: Buffer
+  headers?: Record<string, Buffer>
+}
+export interface ProduceRecord {
+  topic: string
+  messages: Array<MessageModel>
+}
+export const enum AutoOffsetReset {
+  Smallest = 'Smallest',
+  Earliest = 'Earliest',
+  Beginning = 'Beginning',
+  Largest = 'Largest',
+  Latest = 'Latest',
+  End = 'End',
+  Error = 'Error'
+}
+export const enum KafkaCommitMode {
+  AutoCommit = 'AutoCommit',
+  Sync = 'Sync',
+  Async = 'Async'
+}
+export interface ConsumerConfiguration {
+  topic: string
+  groupId: string
+  retryStrategy?: RetryStrategy
+  offset?: OffsetModel
+  createTopic?: boolean
+  commitMode?: KafkaCommitMode
+}
+export const enum PartitionPosition {
+  Beginning = 0,
+  End = 1,
+  Stored = 2
+}
+export interface OffsetModel {
+  offset?: number
+  position?: PartitionPosition
+}
+export interface RetryStrategy {
+  retries: number
+  nextTopicOnFail: string
+  pauseConsumerDuration?: number
+}
+export const enum ConsumerResult {
+  Ok = 'Ok',
+  Retry = 'Retry'
+}
+export class KafkaConfiguration {
+  brokers: string
+  clientId: string
+  securityProtocol?: SecurityProtocol
+  allowAutoCreateTopic?: boolean
+}
+export class KafkaClient {
+  readonly kafkaConfiguration: KafkaConfiguration
+  constructor(brokers: string, clientId: string)
+  createProducer(): KafkaProducer
+  createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
+}
+export class KafkaConsumer {
+  startConsumer(callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult>): Promise<void>
+}
+export class KafkaProducer {
+  send(topic: string, message: MessageModel): Promise<OwnedDelivery>
+}

package/index.js

@@ -0,0 +1,265 @@
+/* tslint:disable */
+/* eslint-disable */
+/* prettier-ignore */
+
+/* auto-generated by NAPI-RS */
+
+const { existsSync, readFileSync } = require('fs')
+const { join } = require('path')
+
+const { platform, arch } = process
+
+let nativeBinding = null
+let localFileExisted = false
+let loadError = null
+
+function isMusl() {
+  // For Node 10
+  if (!process.report || typeof process.report.getReport !== 'function') {
+    try {
+      const lddPath = require('child_process').execSync('which ldd').toString().trim()
+      return readFileSync(lddPath, 'utf8').includes('musl')
+    } catch (e) {
+      return true
+    }
+  } else {
+    const { glibcVersionRuntime } = process.report.getReport().header
+    return !glibcVersionRuntime
+  }
+}
+
+switch (platform) {
+  case 'android':
+    switch (arch) {
+      case 'arm64':
+        localFileExisted = existsSync(join(__dirname, 'kafka-crab-js.android-arm64.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.android-arm64.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-android-arm64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm':
+        localFileExisted = existsSync(join(__dirname, 'kafka-crab-js.android-arm-eabi.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.android-arm-eabi.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-android-arm-eabi')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Android ${arch}`)
+    }
+    break
+  case 'win32':
+    switch (arch) {
+      case 'x64':
+        localFileExisted = existsSync(
+          join(__dirname, 'kafka-crab-js.win32-x64-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.win32-x64-msvc.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-win32-x64-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'ia32':
+        localFileExisted = existsSync(
+          join(__dirname, 'kafka-crab-js.win32-ia32-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.win32-ia32-msvc.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-win32-ia32-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm64':
+        localFileExisted = existsSync(
+          join(__dirname, 'kafka-crab-js.win32-arm64-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.win32-arm64-msvc.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-win32-arm64-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Windows: ${arch}`)
+    }
+    break
+  case 'darwin':
+    localFileExisted = existsSync(join(__dirname, 'kafka-crab-js.darwin-universal.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./kafka-crab-js.darwin-universal.node')
+      } else {
+        nativeBinding = require('kafka-crab-js-darwin-universal')
+      }
+      break
+    } catch {}
+    switch (arch) {
+      case 'x64':
+        localFileExisted = existsSync(join(__dirname, 'kafka-crab-js.darwin-x64.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.darwin-x64.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-darwin-x64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm64':
+        localFileExisted = existsSync(
+          join(__dirname, 'kafka-crab-js.darwin-arm64.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.darwin-arm64.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-darwin-arm64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on macOS: ${arch}`)
+    }
+    break
+  case 'freebsd':
+    if (arch !== 'x64') {
+      throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
+    }
+    localFileExisted = existsSync(join(__dirname, 'kafka-crab-js.freebsd-x64.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./kafka-crab-js.freebsd-x64.node')
+      } else {
+        nativeBinding = require('kafka-crab-js-freebsd-x64')
+      }
+    } catch (e) {
+      loadError = e
+    }
+    break
+  case 'linux':
+    switch (arch) {
+      case 'x64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-x64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-x64-musl.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-x64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-x64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-x64-gnu.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-x64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'arm64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-arm64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-arm64-musl.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-arm64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-arm64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-arm64-gnu.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-arm64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'arm':
+        localFileExisted = existsSync(
+          join(__dirname, 'kafka-crab-js.linux-arm-gnueabihf.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./kafka-crab-js.linux-arm-gnueabihf.node')
+          } else {
+            nativeBinding = require('kafka-crab-js-linux-arm-gnueabihf')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Linux: ${arch}`)
+    }
+    break
+  default:
+    throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`)
+}
+
+if (!nativeBinding) {
+  if (loadError) {
+    throw loadError
+  }
+  throw new Error(`Failed to load native binding`)
+}
+
+const { SecurityProtocol, KafkaConfiguration, KafkaClient, KafkaConsumer, KafkaProducer, AutoOffsetReset, KafkaCommitMode, PartitionPosition, ConsumerResult } = nativeBinding
+
+module.exports.SecurityProtocol = SecurityProtocol
+module.exports.KafkaConfiguration = KafkaConfiguration
+module.exports.KafkaClient = KafkaClient
+module.exports.KafkaConsumer = KafkaConsumer
+module.exports.KafkaProducer = KafkaProducer
+module.exports.AutoOffsetReset = AutoOffsetReset
+module.exports.KafkaCommitMode = KafkaCommitMode
+module.exports.PartitionPosition = PartitionPosition
+module.exports.ConsumerResult = ConsumerResult

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "kafka-crab-js",
+  "version": "0.0.0",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "napi": {
+    "name": "kafka-crab-js",
+    "triples": {
+      "additional": [
+        "aarch64-apple-darwin",
+        "x86_64-unknown-linux-musl"
+      ]
+    }
+  },
+  "license": "MIT",
+  "devDependencies": {
+    "@napi-rs/cli": "^2.16.1",
+    "ava": "^5.1.1",
+    "nanoid": "^4.0.2"
+  },
+  "ava": {
+    "timeout": "3m"
+  },
+  "engines": {
+    "node": ">= 10"
+  },
+  "scripts": {
+    "artifacts": "napi artifacts",
+    "build": "napi build --platform --release",
+    "build:debug": "napi build --platform",
+    "prepublishOnly": "napi prepublish -t npm",
+    "test": "ava",
+    "universal": "napi universal",
+    "version": "napi version"
+  },
+  "packageManager": "yarn@3.6.0",
+  "optionalDependencies": {
+    "kafka-crab-js-win32-x64-msvc": "0.0.0",
+    "kafka-crab-js-darwin-x64": "0.0.0",
+    "kafka-crab-js-linux-x64-gnu": "0.0.0",
+    "kafka-crab-js-darwin-arm64": "0.0.0",
+    "kafka-crab-js-linux-x64-musl": "0.0.0"
+  }
+}

package/src/kafka/kafka_admin.rs

@@ -0,0 +1,129 @@
+use rdkafka::{
+  admin::{
+    AdminClient, AdminOptions, ConfigResource, NewTopic, ResourceSpecifier, TopicReplication,
+  },
+  client::DefaultClientContext,
+  config::ClientConfig,
+  consumer::{BaseConsumer, Consumer},
+  error::KafkaError,
+  types::RDKafkaErrorCode,
+};
+use tracing::{debug, info};
+
+use std::{collections::HashMap, str::FromStr, time::Duration};
+
+const DEFAULT_NUM_PARTITIONS: i32 = 3;
+const DEFAULT_REPLICATION: i32 = 3;
+
+pub struct KafkaAdmin<'a> {
+  client_config: &'a ClientConfig,
+  admin_client: AdminClient<DefaultClientContext>,
+}
+
+impl<'a> KafkaAdmin<'a> {
+  pub fn new(client_config: &'a ClientConfig) -> Self {
+    let admin_client: AdminClient<DefaultClientContext> = client_config
+      .create()
+      .expect("admin client creation failed");
+
+    KafkaAdmin {
+      client_config,
+      admin_client,
+    }
+  }
+
+  async fn fetch_config_resource(&self) -> Result<HashMap<String, String>, KafkaError> {
+    let consumer: BaseConsumer = self
+      .client_config
+      .create()
+      .expect("Consumer creation failed");
+
+    let metadata = consumer
+      .fetch_metadata(None, Duration::from_secs(5))
+      .expect("Failed to fetch metadata");
+
+    for broker in metadata.brokers() {
+      info!(
+        "Id: {}  Host: {}:{}  ",
+        broker.id(),
+        broker.host(),
+        broker.port()
+      );
+    }
+
+    let config_resource = self
+      .admin_client
+      .describe_configs(
+        &[ResourceSpecifier::Broker(metadata.orig_broker_id())],
+        &AdminOptions::new(),
+      )
+      .await?;
+
+    Ok(extract_config_resource(config_resource))
+  }
+
+  pub async fn create_topic(&self, topic_name: &str) -> anyhow::Result<()> {
+    let broker_properties = self.fetch_config_resource().await?.clone();
+
+    debug!("Broker properties {:?}", broker_properties);
+
+    self
+      .admin_client
+      .create_topics(
+        &[NewTopic {
+          name: topic_name,
+          num_partitions: broker_properties
+            .get("num.partitions")
+            .get_parsed_or_default_value(DEFAULT_NUM_PARTITIONS),
+          replication: TopicReplication::Fixed(
+            broker_properties
+              .get("default.replication.factor")
+              .get_parsed_or_default_value(DEFAULT_REPLICATION),
+          ),
+          config: vec![],
+        }],
+        &AdminOptions::default(),
+      )
+      .await
+      .map_err(anyhow::Error::new)?;
+
+    info!("Topic {} was created successfully", topic_name);
+    Ok(())
+  }
+}
+
+fn extract_config_resource(
+  config_resource: Vec<Result<ConfigResource, RDKafkaErrorCode>>,
+) -> HashMap<String, String> {
+  let mut properties: HashMap<String, String> = HashMap::new();
+  for config_resource_list in config_resource {
+    match config_resource_list {
+      Ok(v) => {
+        for config_entry_list in v.entries {
+          if let Some(value) = config_entry_list.value {
+            properties.insert(config_entry_list.name, value);
+          }
+        }
+      }
+      Err(e) => {
+        debug!("Error on fetching config entry {:?}", e)
+      }
+    }
+  }
+
+  properties
+}
+
+trait ParsedOrDefaultValue {
+  fn get_parsed_or_default_value<T: FromStr>(self, default_value: T) -> T;
+}
+
+impl ParsedOrDefaultValue for Option<&String> {
+  fn get_parsed_or_default_value<T: FromStr>(self, default_value: T) -> T {
+    if let Some(ret) = self {
+      ret.parse().unwrap_or(default_value)
+    } else {
+      default_value
+    }
+  }
+}

package/src/kafka/kafka_client.rs

@@ -0,0 +1,131 @@
+use std::fmt;
+
+use napi::bindgen_prelude::*;
+use rdkafka::{
+  client::ClientContext,
+  config::{ClientConfig, RDKafkaLogLevel},
+  consumer::{ConsumerContext, Rebalance},
+  error::KafkaResult,
+  topic_partition_list::TopicPartitionList,
+};
+use tracing::info;
+
+use super::{
+  kafka_consumer::KafkaConsumer, kafka_producer::KafkaProducer, model::ConsumerConfiguration,
+};
+
+struct CustomContext;
+
+impl ClientContext for CustomContext {}
+
+impl ConsumerContext for CustomContext {
+  fn pre_rebalance(&self, rebalance: &Rebalance) {
+    info!("Pre rebalance {:?}", rebalance);
+  }
+
+  fn post_rebalance(&self, rebalance: &Rebalance) {
+    info!("Post rebalance {:?}", rebalance);
+  }
+
+  fn commit_callback(&self, result: KafkaResult<()>, _offsets: &TopicPartitionList) {
+    info!("Committing offsets: {:?}. Offset: {:?}", result, _offsets);
+  }
+}
+
+#[derive(Debug)]
+#[napi(string_enum)]
+pub enum SecurityProtocol {
+  Plaintext,
+  Ssl,
+  SaslPlaintext,
+  SaslSsl,
+}
+
+impl fmt::Display for SecurityProtocol {
+  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    match self {
+      SecurityProtocol::Plaintext => write!(f, "plaintext"),
+      SecurityProtocol::Ssl => write!(f, "ssl"),
+      SecurityProtocol::SaslPlaintext => write!(f, "sasl_plaintext"),
+      SecurityProtocol::SaslSsl => write!(f, "sasl_ssl"),
+    }
+  }
+}
+
+#[derive(Clone, Debug)]
+#[napi]
+pub struct KafkaConfiguration {
+  pub brokers: String,
+  pub client_id: String,
+  pub security_protocol: Option<SecurityProtocol>,
+  pub allow_auto_create_topic: Option<bool>,
+}
+
+#[derive(Clone, Debug)]
+#[napi]
+pub struct KafkaClient {
+  rdkafka_client_config: ClientConfig,
+  #[napi(readonly)]
+  pub kafka_configuration: KafkaConfiguration,
+}
+
+#[napi]
+impl KafkaClient {
+  #[napi(constructor)]
+  pub fn new(brokers: String, client_id: String) -> Self {
+    match tracing_subscriber::fmt::try_init() {
+      Ok(_) => {}
+      Err(e) => println!("Failed to initialize tracing: {:?}", e),
+    }
+
+    KafkaClient::with_kafka_configuration(KafkaConfiguration {
+      brokers,
+      client_id,
+      security_protocol: None,
+      allow_auto_create_topic: None,
+    })
+  }
+
+  pub fn get_client_config(&self) -> &ClientConfig {
+    &self.rdkafka_client_config
+  }
+
+  pub fn with_kafka_configuration(kafka_configuration: KafkaConfiguration) -> Self {
+    let KafkaConfiguration {
+      brokers,
+      security_protocol,
+      client_id,
+      allow_auto_create_topic,
+    } = kafka_configuration.clone();
+
+    let mut rdkafka_client_config = ClientConfig::new();
+
+    rdkafka_client_config.set_log_level(RDKafkaLogLevel::Info);
+    rdkafka_client_config.set("bootstrap.servers", brokers);
+    rdkafka_client_config.set("client.id", client_id);
+    if let Some(security_protocol) = security_protocol {
+      rdkafka_client_config.set("security.protocol", security_protocol.to_string());
+    }
+    if let Some(allow_auto_create_topic) = allow_auto_create_topic {
+      rdkafka_client_config.set(
+        "allow.auto.create.topics",
+        allow_auto_create_topic.to_string(),
+      );
+    }
+
+    KafkaClient {
+      rdkafka_client_config,
+      kafka_configuration,
+    }
+  }
+
+  #[napi]
+  pub fn create_producer(&self) -> KafkaProducer {
+    KafkaProducer::new(self.rdkafka_client_config.clone())
+  }
+
+  #[napi]
+  pub fn create_consumer(&self, consumer_configuration: ConsumerConfiguration) -> KafkaConsumer {
+    KafkaConsumer::new(self.clone(), consumer_configuration)
+  }
+}

package/src/kafka/kafka_consumer.rs

@@ -0,0 +1,188 @@
+use napi::{
+  bindgen_prelude::{Buffer, Promise},
+  threadsafe_function::ThreadsafeFunction,
+  Result,
+};
+use rdkafka::{
+  client::ClientContext,
+  config::RDKafkaLogLevel,
+  consumer::{stream_consumer::StreamConsumer, CommitMode, Consumer, ConsumerContext, Rebalance},
+  error::KafkaResult,
+  message::BorrowedMessage,
+  topic_partition_list::TopicPartitionList,
+  Message, Offset,
+};
+
+use std::{time::Duration, vec};
+use tracing::{debug, error, info};
+
+use super::{
+  kafka_admin::KafkaAdmin,
+  kafka_client::KafkaClient,
+  model::{ConsumerConfiguration, ConsumerResult, KafkaCommitMode, OffsetModel, PartitionPosition},
+};
+
+type LoggingConsumer = StreamConsumer<CustomContext>;
+
+pub struct CustomContext;
+
+impl ClientContext for CustomContext {}
+
+impl ConsumerContext for CustomContext {
+  fn pre_rebalance(&self, rebalance: &Rebalance) {
+    info!("Pre rebalance {:?}", rebalance);
+  }
+
+  fn post_rebalance(&self, rebalance: &Rebalance) {
+    info!("Post rebalance {:?}", rebalance);
+  }
+
+  fn commit_callback(&self, result: KafkaResult<()>, _offsets: &TopicPartitionList) {
+    info!("Committing offsets: {:?}. Offset: {:?}", result, _offsets);
+  }
+}
+
+#[derive(Clone, Debug)]
+#[napi]
+pub struct KafkaConsumer {
+  kafka_client: KafkaClient,
+  consumer_configuration: ConsumerConfiguration,
+}
+
+#[napi]
+impl KafkaConsumer {
+  pub fn new(kafka_client: KafkaClient, consumer_configuration: ConsumerConfiguration) -> Self {
+    KafkaConsumer {
+      kafka_client,
+      consumer_configuration,
+    }
+  }
+
+  #[napi(ts_args_type = "callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult>")]
+  pub async fn start_consumer(&self, func: ThreadsafeFunction<Buffer>) -> Result<()> {
+    let ConsumerConfiguration { commit_mode, .. } = self.consumer_configuration.clone();
+
+    let commit_mode = match commit_mode {
+      None | Some(KafkaCommitMode::AutoCommit) => None, //Default AutoCommit
+      Some(KafkaCommitMode::Sync) => Some(CommitMode::Sync),
+      Some(KafkaCommitMode::Async) => Some(CommitMode::Async),
+    };
+
+    let stream_consumer: StreamConsumer<CustomContext> = self
+      .setup_consumer()
+      .await
+      .expect("Consumer creation failed");
+
+    tokio::spawn(async move {
+      loop {
+        match stream_consumer.recv().await {
+          Err(e) => error!("Error while receiving from stream consumer: {:?}", e),
+          Ok(message) => process_message(commit_mode, &stream_consumer, &func, &message).await,
+        };
+      }
+    });
+
+    Ok(())
+  }
+
+  async fn setup_consumer(&self) -> anyhow::Result<StreamConsumer<CustomContext>> {
+    let context = CustomContext;
+
+    let ConsumerConfiguration {
+      topic,
+      offset,
+      create_topic,
+      group_id,
+      ..
+    } = self.consumer_configuration.clone();
+
+    let consumer: LoggingConsumer = self
+      .kafka_client
+      .get_client_config()
+      .clone()
+      .set("group.id", group_id.clone())
+      .set("enable.auto.commit", "false")
+      .set_log_level(RDKafkaLogLevel::Debug)
+      .create_with_context(context)
+      .expect("Consumer creation failed");
+
+    if create_topic.unwrap_or(true) {
+      let admin = KafkaAdmin::new(self.kafka_client.get_client_config());
+      admin.create_topic(&topic).await?;
+    }
+
+    if let Some(offset) = convert_to_rdkafka_offset(offset) {
+      let metadata = consumer
+        .fetch_metadata(Some(&topic), Duration::from_millis(1500))
+        .expect("Fail to retrive metadata from consumer");
+
+      metadata.topics().iter().for_each(|meta_topic| {
+        let mut tpl = TopicPartitionList::new();
+        meta_topic.partitions().iter().for_each(|meta_partition| {
+          tpl.add_partition(&topic, meta_partition.id());
+        });
+        tpl.set_all_offsets(offset).expect("Fail to set offset");
+        consumer
+          .assign(&tpl)
+          .expect("Assign topic partition list failed");
+      });
+    }
+
+    consumer
+      .subscribe(vec![&*topic.to_string()].as_slice())
+      .expect("Can't subscribe to specified topics");
+
+    info!(
+      "Consumer created. Group id: {:?}, Topic: {:?}",
+      group_id, topic
+    );
+    Ok(consumer)
+  }
+}
+
+fn convert_to_rdkafka_offset(offset_model: Option<OffsetModel>) -> Option<Offset> {
+  match offset_model {
+    Some(model) => Some(match model.position {
+      Some(PartitionPosition::Beginning) => Offset::Beginning,
+      Some(PartitionPosition::End) => Offset::End,
+      Some(PartitionPosition::Stored) => Offset::Stored,
+      None => match model.offset {
+        Some(value) => Offset::Offset(value),
+        None => Offset::Stored, // Default to stored
+      },
+    }),
+    None => None,
+  }
+}
+
+async fn process_message(
+  commit_mode: Option<CommitMode>,
+  stream_consumer: &StreamConsumer<CustomContext>,
+  func: &ThreadsafeFunction<Buffer>,
+  message: &BorrowedMessage<'_>,
+) {
+  match message.payload_view::<[u8]>() {
+    None => {}
+    Some(Ok(payload)) => match func
+      .call_async::<Promise<ConsumerResult>>(Ok(payload.into()))
+      .await
+    {
+      Ok(js_result) => match js_result.await {
+        Ok(value) => {
+          debug!("Message processed: {:?}", value);
+          if let Some(commit_mode) = commit_mode {
+            match stream_consumer.commit_message(&message, commit_mode) {
+              Ok(_) => debug!("Message committed"),
+              Err(e) => error!("Error on commit: {:?}", e),
+            }
+          }
+        }
+        Err(e) => error!("Error on promise: {:?}", e),
+      },
+      Err(js_error) => error!("Error on js call: {:?}", js_error),
+    },
+    Some(Err(e)) => {
+      error!("Error while deserializing message payload: {:?}", e);
+    }
+  }
+}

package/src/kafka/kafka_producer.rs

@@ -0,0 +1,57 @@
+use std::time::Duration;
+
+use napi::{Error, Result, Status};
+use rdkafka::{
+  message::ToBytes,
+  producer::{FutureProducer, FutureRecord},
+  ClientConfig,
+};
+
+use super::model::MessageModel;
+
+#[derive(Clone)]
+#[napi]
+pub struct KafkaProducer {
+  future_producer: FutureProducer,
+}
+
+#[napi(object)]
+pub struct OwnedDelivery {
+  pub partition: i32,
+  pub offset: i64,
+}
+
+#[napi]
+impl KafkaProducer {
+  pub fn new(client_config: ClientConfig) -> Self {
+    let future_producer = client_config
+      .clone()
+      .set("message.timeout.ms", "5000")
+      .create::<FutureProducer>()
+      .expect("Producer creation error");
+
+    KafkaProducer { future_producer }
+  }
+
+  #[napi]
+  pub async fn send(&self, topic: String, message: MessageModel) -> Result<OwnedDelivery> {
+    let MessageModel {
+      key,
+      value,
+      headers: _,
+    } = message;
+
+    let record = FutureRecord::to(&topic)
+      .key(key.to_bytes())
+      .payload(value.to_bytes());
+
+    match self
+      .future_producer
+      .send(record, Duration::from_secs(5000))
+      .await
+    {
+      Ok((partition, offset)) => Ok(OwnedDelivery { partition, offset }),
+      Err((kafka_error, _)) => Err(Error::new(Status::GenericFailure, kafka_error)),
+    }
+  }
+}

package/src/kafka/kafka_util.rs

@@ -0,0 +1,78 @@
+use std::collections::HashMap;
+
+use rdkafka::message::{BorrowedHeaders, Headers};
+
+pub fn convert_kakfa_headers_to_hashmap(headers: Option<&BorrowedHeaders>) -> HashMap<&str, &[u8]> {
+  match headers {
+    Some(value) => value
+      .iter()
+      .filter(|it| it.value.is_some())
+      .map(|it| (it.key, it.value.unwrap()))
+      .collect::<HashMap<&str, &[u8]>>(),
+    _ => HashMap::new(),
+  }
+}
+
+pub trait ExtractValueOnKafkaHashMap<T> {
+  fn get_value(&self, key: &str) -> Option<T>;
+}
+
+impl ExtractValueOnKafkaHashMap<usize> for HashMap<&str, &[u8]> {
+  fn get_value(&self, key: &str) -> Option<usize> {
+    match self.get(key) {
+      Some(it) => {
+        let parsed = match String::from_utf8(it.to_vec()) {
+          Ok(it) => Some(it.parse::<usize>()),
+          Err(_) => None,
+        };
+        match parsed {
+          Some(value) => value.ok(),
+          _ => None,
+        }
+      }
+      None => None,
+    }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use std::collections::HashMap;
+
+  use rdkafka::message::{Header, OwnedHeaders};
+
+  use crate::kafka::kafka_util::{convert_kakfa_headers_to_hashmap, ExtractValueOnKafkaHashMap};
+
+  fn create_header<'a>(key: &'a str, value: Option<&'a str>) -> Header<'a, &'a str> {
+    Header { key, value }
+  }
+
+  #[test]
+  fn filter_out_nones() {
+    let _x = Header {
+      key: "header_key",
+      value: Some("header_value"),
+    };
+    let headers = OwnedHeaders::new()
+      .insert(create_header("key1", Some("1")))
+      .insert(create_header("key2", None))
+      .insert(create_header("key4", None))
+      .insert(create_header("key5", Some("5")));
+
+    let hash_map = convert_kakfa_headers_to_hashmap(Some(headers.as_borrowed()));
+
+    assert_eq!(hash_map.len(), 2);
+  }
+
+  #[test]
+  fn extract_usize() {
+    let hash_map = HashMap::from([("key1", "32".as_bytes())]);
+    assert_eq!(hash_map.get_value("key1"), Some(32));
+  }
+
+  #[test]
+  fn extract_empty_value() {
+    let hash_map = HashMap::from([("key1", "32".as_bytes())]);
+    assert_eq!(hash_map.get_value("key2"), None);
+  }
+}

package/src/kafka/mod.rs

@@ -0,0 +1,6 @@
+pub mod kafka_admin;
+pub mod kafka_client;
+pub mod kafka_consumer;
+pub mod kafka_producer;
+pub mod kafka_util;
+pub mod model;

package/src/kafka/model.rs

@@ -0,0 +1,86 @@
+use napi::bindgen_prelude::*;
+
+use std::{collections::HashMap, fmt};
+
+use napi::bindgen_prelude::Buffer;
+
+#[napi(object)]
+#[derive(Clone)]
+pub struct MessageModel {
+  pub key: Buffer,
+  pub value: Buffer,
+  pub headers: Option<HashMap<String, Buffer>>,
+}
+
+#[napi(object)]
+#[derive(Clone)]
+pub struct ProduceRecord {
+  pub topic: String,
+  pub messages: Vec<MessageModel>,
+}
+
+#[derive(Debug)]
+#[napi(string_enum)]
+pub enum AutoOffsetReset {
+  Smallest,
+  Earliest,
+  Beginning,
+  Largest,
+  Latest,
+  End,
+  Error,
+}
+
+impl fmt::Display for AutoOffsetReset {
+  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    write!(f, "{}", format!("{:?}", self).to_lowercase())
+  }
+}
+
+#[napi(string_enum)]
+#[derive(Debug)]
+pub enum KafkaCommitMode {
+  AutoCommit,
+  Sync,
+  Async,
+}
+
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct ConsumerConfiguration {
+  pub topic: String,
+  pub group_id: String,
+  pub retry_strategy: Option<RetryStrategy>,
+  pub offset: Option<OffsetModel>,
+  pub create_topic: Option<bool>,
+  pub commit_mode: Option<KafkaCommitMode>,
+}
+
+#[napi(object)]
+#[derive(Debug)]
+pub enum PartitionPosition {
+  Beginning,
+  End,
+  Stored,
+}
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct OffsetModel {
+  pub offset: Option<i64>,
+  pub position: Option<PartitionPosition>,
+}
+
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct RetryStrategy {
+  pub retries: i32,
+  pub next_topic_on_fail: String,
+  pub pause_consumer_duration: Option<i64>,
+}
+
+#[napi(string_enum)]
+#[derive(Debug)]
+pub enum ConsumerResult {
+  Ok,
+  Retry,
+}

package/src/lib.rs

@@ -0,0 +1,6 @@
+#![deny(clippy::all)]
+
+mod kafka;
+
+#[macro_use]
+extern crate napi_derive;