kafka-crab-js 0.1.2 → 0.2.0

package/Cargo.toml

@@ -26,7 +26,7 @@ rdkafka = { version = "0.33", features = [
 tokio = { version = "1", features = ["full"] }
 
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+tracing-subscriber = { version = "0.3", features = ["json"] }
 
 [build-dependencies]
 napi-build = "2.0.1"

package/example/sample.mjs

@@ -7,8 +7,8 @@ import {KafkaClient, CommitMode, PartitionPosition, ConsumerResult} from '../ind
 
 const kafkaClient = new KafkaClient({
   brokers: 'localhost:29092',
-  clientId: 'my-ids',
-  groupId: 'my-js-group',
+  clientId: 'my-js-group',
+  logLevel: 'info',
   enableAnsiLogger: true});
 const topic = `topic-${nanoid()}`;
 

package/index.d.ts

@@ -14,7 +14,7 @@ export interface KafkaConfiguration {
   clientId: string
   securityProtocol?: SecurityProtocol
   configuration?: Record<string, string>
-  enableAnsiLogger?: boolean
+  logLevel?: string
 }
 export const enum CommitMode {
   AutoCommit = 'AutoCommit',
@@ -26,6 +26,8 @@ export interface RetryStrategy {
   retryTopic?: string
   dqlTopic?: string
   pauseConsumerDuration?: number
+  offset?: OffsetModel
+  configuration?: Record<string, string>
 }
 export const enum ConsumerResult {
   Ok = 'Ok',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -55,8 +55,8 @@
   },
   "packageManager": "yarn@3.6.0",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.1.2",
-    "kafka-crab-js-darwin-x64": "0.1.2",
-    "kafka-crab-js-linux-x64-gnu": "0.1.2"
+    "kafka-crab-js-darwin-arm64": "0.2.0",
+    "kafka-crab-js-darwin-x64": "0.2.0",
+    "kafka-crab-js-linux-x64-gnu": "0.2.0"
   }
 }

package/src/kafka/kafka_client.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, fmt};
+use std::{collections::HashMap, fmt, str::FromStr};
 
 use napi::{bindgen_prelude::*, Result};
 use rdkafka::{
@@ -9,8 +9,7 @@ use rdkafka::{
   topic_partition_list::TopicPartitionList,
 };
 
-use tracing::info;
-use tracing_subscriber::EnvFilter;
+use tracing::{info, Level};
 
 use super::{
   kafka_consumer::{ConsumerConfiguration, KafkaConsumer},
@@ -62,7 +61,7 @@ pub struct KafkaConfiguration {
   pub client_id: String,
   pub security_protocol: Option<SecurityProtocol>,
   pub configuration: Option<HashMap<String, String>>,
-  pub enable_ansi_logger: Option<bool>,
+  pub log_level: Option<String>,
 }
 
 #[derive(Clone, Debug)]
@@ -77,9 +76,13 @@ pub struct KafkaClient {
 impl KafkaClient {
   #[napi(constructor)]
   pub fn new(kafka_configuration: KafkaConfiguration) -> Self {
+    let log_level = kafka_configuration.clone().log_level;
     match tracing_subscriber::fmt()
-      .with_ansi(kafka_configuration.enable_ansi_logger.unwrap_or(false))
-      .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
+      .with_max_level(
+        Level::from_str(log_level.unwrap_or("error".to_owned()).as_str()).unwrap_or(Level::ERROR),
+      )
+      .json()
+      .with_ansi(false)
       .try_init()
     {
       Ok(_) => {}

package/src/kafka/kafka_consumer.rs

@@ -25,7 +25,7 @@ use super::{
   kafka_admin::KafkaAdmin,
   kafka_client::KafkaClient,
   kafka_util::{kakfa_headers_to_hashmap_buffer, AnyhowToNapiError},
-  model::{OffsetModel, Payload},
+  model::{OffsetModel, PartitionPosition, Payload},
 };
 
 const RETRY_COUNTER_NAME: &str = "kafka-crab-js-retry-counter";
@@ -69,6 +69,8 @@ pub struct RetryStrategy {
   pub retry_topic: Option<String>,
   pub dql_topic: Option<String>,
   pub pause_consumer_duration: Option<i64>,
+  pub offset: Option<OffsetModel>,
+  pub configuration: Option<HashMap<String, String>>,
 }
 
 #[napi(string_enum)]
@@ -184,7 +186,11 @@ impl KafkaConsumer {
     };
     let consumer_thread = ConsumerThread {
       stream_consumer: self
-        .create_stream_consumer(&self.topic)
+        .create_stream_consumer(
+          &self.topic,
+          self.consumer_configuration.offset.clone(),
+          self.consumer_configuration.configuration.clone(),
+        )
         .await
         .map_err(|e| e.convert_to_napi())?,
       func: func.clone(),
@@ -223,9 +229,17 @@ impl KafkaConsumer {
         .pause_consumer_duration
         .unwrap_or(DEFAULT_PAUSE_DURATION) as u64,
     ));
+    let offset_model = strategy.offset.clone().unwrap_or(OffsetModel {
+      position: Some(PartitionPosition::Stored),
+      offset: None,
+    });
     let consumer_thread = ConsumerThread {
       stream_consumer: self
-        .create_stream_consumer(&retry_topic)
+        .create_stream_consumer(
+          &retry_topic,
+          Some(offset_model),
+          strategy.configuration.clone(),
+        )
         .await
         .map_err(|e| e.convert_to_napi())?,
       func: func.clone(),
@@ -246,14 +260,14 @@ impl KafkaConsumer {
   async fn create_stream_consumer(
     &self,
     topic: &str,
+    offset: Option<OffsetModel>,
+    configuration: Option<HashMap<String, String>>,
   ) -> anyhow::Result<StreamConsumer<CustomContext>> {
     let context = CustomContext;
 
     let ConsumerConfiguration {
-      offset,
       create_topic,
       group_id,
-      configuration,
       enable_auto_commit,
       ..
     } = self.consumer_configuration.clone();
@@ -339,7 +353,8 @@ impl ConsumerThread {
           match self.handle_message_result(&message, message_result).await {
             Ok(_) => {
               debug!(
-                "Message consumed successfully. Partition: {}, Offset: {}",
+                "Message consumed successfully. Topic: {}, Partition: {}, Offset: {}",
+                message.topic(),
                 message.partition(),
                 message.offset()
               );