kafka-crab-js 0.1.2 → 0.3.0

package/Cargo.toml

@@ -26,7 +26,7 @@ rdkafka = { version = "0.33", features = [
 tokio = { version = "1", features = ["full"] }
 
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+tracing-subscriber = { version = "0.3", features = ["json"] }
 
 [build-dependencies]
 napi-build = "2.0.1"

package/README.md

@@ -1,7 +1,5 @@
 # Kafka Crab Js
 
-🚧 Project in Alpha Stage: API Subject to Breaking Changes 🚧
-
 Kafka-Crab-JS is a powerful Node.js library that allows developers to interact with Apache Kafka using Rust programming language, seamlessly integrated with Node.js applications. This documentation provides an overview of the Kafka-Crab-JS library, including installation instructions, usage guidelines, and examples.
 
 The plugin can be found on [npm](https://www.npmjs.com/package/kafka-crab-js) and the source code is available on [GitHub](https://github.com/inaiat/kafka-crab-js).

package/example/sample.mjs

@@ -1,29 +1,29 @@
 /* eslint-disable no-await-in-loop */
 
 import timersPromises from 'node:timers/promises';
-import {Buffer} from 'node:buffer';
-import {nanoid} from 'nanoid';
-import {KafkaClient, CommitMode, PartitionPosition, ConsumerResult} from '../index.js';
+import { Buffer } from 'node:buffer';
+import { nanoid } from 'nanoid';
+import { KafkaClient, CommitMode, PartitionPosition, ConsumerResult } from '../index.js';
 
 const kafkaClient = new KafkaClient({
   brokers: 'localhost:29092',
-  clientId: 'my-ids',
-  groupId: 'my-js-group',
-  enableAnsiLogger: true});
+  clientId: 'my-js-group',
+  logLevel: 'info',
+});
 const topic = `topic-${nanoid()}`;
 
 const consumer = kafkaClient.createConsumer({
   topic,
   groupId: 'my-js-group',
-  CommitMode: CommitMode.AutoCommit,
-  offset: {position: PartitionPosition.Stored},
-  configuration: {'auto.offset.reset': 'earliest'},
+  commitMode: CommitMode.AutoCommit,
+  offset: { position: PartitionPosition.Stored },
+  configuration: { 'auto.offset.reset': 'earliest' },
   retryStrategy: {
     retries: 5,
     pauseConsumerDuration: 5000,
   },
 });
-const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '5000'}});
+const producer = kafkaClient.createProducer({ topic, configuration: { 'message.timeout.ms': '5000' } });
 
 async function produce() {
   for (let i = 0; i < 10; i++) {
@@ -31,7 +31,7 @@ async function produce() {
       const result = await producer.send(
         {
           topic,
-          messages: [{value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`)}],
+          messages: [{ value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`) }],
         },
       );
       console.log('Js message sent. Offset:', result);
@@ -42,7 +42,7 @@ async function produce() {
 }
 
 async function startConsumer() {
-  consumer.startConsumer(async (error, {value, partition, offset}) => {
+  consumer.startConsumer(async (error, { value, partition, offset }) => {
     if (error) {
       console.error('Js Consumer error', error);
       return;

package/index.d.ts

@@ -14,7 +14,7 @@ export interface KafkaConfiguration {
   clientId: string
   securityProtocol?: SecurityProtocol
   configuration?: Record<string, string>
-  enableAnsiLogger?: boolean
+  logLevel?: string
 }
 export const enum CommitMode {
   AutoCommit = 'AutoCommit',
@@ -26,6 +26,8 @@ export interface RetryStrategy {
   retryTopic?: string
   dqlTopic?: string
   pauseConsumerDuration?: number
+  offset?: OffsetModel
+  configuration?: Record<string, string>
 }
 export const enum ConsumerResult {
   Ok = 'Ok',
@@ -99,7 +101,7 @@ export class KafkaClient {
 }
 export class ProducerHelper { }
 export class KafkaConsumer {
-  startConsumer(callback: (err: Error | null, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
+  startConsumer(callback: (error: Error | undefined, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
 }
 export class KafkaProducer {
   send(producerRecord: ProducerRecord): Promise<Array<RecordMetadata>>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.1.2",
+  "version": "0.3.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -41,7 +41,8 @@
   "xo": {
     "space": true,
     "rules": {
-      "new-cap": "off"
+      "new-cap": "off",
+      "object-curly-spacing": "off"
     }
   },
   "scripts": {
@@ -55,8 +56,8 @@
   },
   "packageManager": "yarn@3.6.0",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.1.2",
-    "kafka-crab-js-darwin-x64": "0.1.2",
-    "kafka-crab-js-linux-x64-gnu": "0.1.2"
+    "kafka-crab-js-darwin-arm64": "0.3.0",
+    "kafka-crab-js-darwin-x64": "0.3.0",
+    "kafka-crab-js-linux-x64-gnu": "0.3.0"
   }
 }

package/src/kafka/kafka_client.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, fmt};
+use std::{collections::HashMap, fmt, str::FromStr};
 
 use napi::{bindgen_prelude::*, Result};
 use rdkafka::{
@@ -9,8 +9,7 @@ use rdkafka::{
   topic_partition_list::TopicPartitionList,
 };
 
-use tracing::info;
-use tracing_subscriber::EnvFilter;
+use tracing::{info, Level};
 
 use super::{
   kafka_consumer::{ConsumerConfiguration, KafkaConsumer},
@@ -62,7 +61,7 @@ pub struct KafkaConfiguration {
   pub client_id: String,
   pub security_protocol: Option<SecurityProtocol>,
   pub configuration: Option<HashMap<String, String>>,
-  pub enable_ansi_logger: Option<bool>,
+  pub log_level: Option<String>,
 }
 
 #[derive(Clone, Debug)]
@@ -77,9 +76,13 @@ pub struct KafkaClient {
 impl KafkaClient {
   #[napi(constructor)]
   pub fn new(kafka_configuration: KafkaConfiguration) -> Self {
+    let log_level = kafka_configuration.clone().log_level;
     match tracing_subscriber::fmt()
-      .with_ansi(kafka_configuration.enable_ansi_logger.unwrap_or(false))
-      .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
+      .with_max_level(
+        Level::from_str(log_level.unwrap_or("error".to_owned()).as_str()).unwrap_or(Level::ERROR),
+      )
+      .json()
+      .with_ansi(false)
       .try_init()
     {
       Ok(_) => {}

package/src/kafka/kafka_consumer.rs

@@ -25,7 +25,7 @@ use super::{
   kafka_admin::KafkaAdmin,
   kafka_client::KafkaClient,
   kafka_util::{kakfa_headers_to_hashmap_buffer, AnyhowToNapiError},
-  model::{OffsetModel, Payload},
+  model::{OffsetModel, PartitionPosition, Payload},
 };
 
 const RETRY_COUNTER_NAME: &str = "kafka-crab-js-retry-counter";
@@ -69,6 +69,8 @@ pub struct RetryStrategy {
   pub retry_topic: Option<String>,
   pub dql_topic: Option<String>,
   pub pause_consumer_duration: Option<i64>,
+  pub offset: Option<OffsetModel>,
+  pub configuration: Option<HashMap<String, String>>,
 }
 
 #[napi(string_enum)]
@@ -148,7 +150,7 @@ impl KafkaConsumer {
   }
 
   #[napi(
-    ts_args_type = "callback: (err: Error | null, result: Payload) => Promise<ConsumerResult | undefined>"
+    ts_args_type = "callback: (error: Error | undefined, result: Payload) => Promise<ConsumerResult | undefined>"
   )]
   pub async fn start_consumer(&self, func: ThreadsafeFunction<Payload>) -> Result<()> {
     let ConsumerConfiguration { retry_strategy, .. } = self.consumer_configuration.clone();
@@ -184,7 +186,11 @@ impl KafkaConsumer {
     };
     let consumer_thread = ConsumerThread {
       stream_consumer: self
-        .create_stream_consumer(&self.topic)
+        .create_stream_consumer(
+          &self.topic,
+          self.consumer_configuration.offset.clone(),
+          self.consumer_configuration.configuration.clone(),
+        )
         .await
         .map_err(|e| e.convert_to_napi())?,
       func: func.clone(),
@@ -223,9 +229,17 @@ impl KafkaConsumer {
         .pause_consumer_duration
         .unwrap_or(DEFAULT_PAUSE_DURATION) as u64,
     ));
+    let offset_model = strategy.offset.clone().unwrap_or(OffsetModel {
+      position: Some(PartitionPosition::Stored),
+      offset: None,
+    });
     let consumer_thread = ConsumerThread {
       stream_consumer: self
-        .create_stream_consumer(&retry_topic)
+        .create_stream_consumer(
+          &retry_topic,
+          Some(offset_model),
+          strategy.configuration.clone(),
+        )
         .await
         .map_err(|e| e.convert_to_napi())?,
       func: func.clone(),
@@ -246,14 +260,14 @@ impl KafkaConsumer {
   async fn create_stream_consumer(
     &self,
     topic: &str,
+    offset: Option<OffsetModel>,
+    configuration: Option<HashMap<String, String>>,
   ) -> anyhow::Result<StreamConsumer<CustomContext>> {
     let context = CustomContext;
 
     let ConsumerConfiguration {
-      offset,
       create_topic,
       group_id,
-      configuration,
       enable_auto_commit,
       ..
     } = self.consumer_configuration.clone();
@@ -339,7 +353,8 @@ impl ConsumerThread {
           match self.handle_message_result(&message, message_result).await {
             Ok(_) => {
               debug!(
-                "Message consumed successfully. Partition: {}, Offset: {}",
+                "Message consumed successfully. Topic: {}, Partition: {}, Offset: {}",
+                message.topic(),
                 message.partition(),
                 message.offset()
               );