kafka-crab-js 0.0.3 → 0.0.9

package/Cargo.toml

@@ -8,16 +8,24 @@ crate-type = ["cdylib"]
 
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2.13", default-features = false, features = ["napi4", "tokio_rt"] }
+napi = { version = "2.13", default-features = false, features = [
+    "napi4",
+    "tokio_rt",
+] }
 napi-derive = "2.13"
 
-anyhow = { version= "1.0", features = ["backtrace"]}
+anyhow = { version = "1.0", features = ["backtrace"] }
 
-rdkafka = { version = "0.32", features = ["libz-static", "cmake-build", "tokio", "ssl-vendored"] }
+rdkafka = { version = "0.32", features = [
+    "libz-static",
+    "cmake-build",
+    "tokio",
+    "ssl-vendored",
+] }
 tokio = { version = "1", features = ["full"] }
 
-tracing            = "0.1"
-tracing-subscriber = "0.3"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 
 [build-dependencies]
 napi-build = "2.0.1"

package/example/sample.mjs

@@ -3,14 +3,17 @@ import timersPromises from 'node:timers/promises';
 import {Buffer} from 'node:buffer';
 import {ConsumerResult, KafkaClient, KafkaCommitMode, PartitionPosition} from '../index.js';
 
-const kafkaClient = new KafkaClient('localhost:29092', 'my-id');
+const kafkaClient = new KafkaClient({ 
+  brokers: 'localhost:29092', 
+  clientId: 'my-id', 
+  enableAnsiLogger: false});
 const topic = 'my-js-topic';
 
 let counter = 0;
 
 async function process(message) {
   await timersPromises.setTimeout(1);
-  return 'message received: ' + message;
+  return 'message received: ' + JSON.stringify(message);
 }
 
 const consumer = kafkaClient.createConsumer({
@@ -27,7 +30,7 @@ console.time('consumer');
 
 consumer.startConsumer(async (error, value) => {
   if (error) {
-    console.error('Consumer error', error);
+    console.error('Js Consumer error', error);
     return;
   }
 
@@ -36,19 +39,20 @@ consumer.startConsumer(async (error, value) => {
   counter++;
   if (counter >= 0) {
     console.timeEnd('consumer');
-    console.log('Counter:', counter, 'Content:', JSON.stringify(content));
+    console.log('Js Counter:', counter, 'Content:', JSON.stringify(content));
     console.time('consumer');
   }
 
   if (message._id === '3') {
-    console.error('Error on process message, let\'s retry', message);
+    console.error('Js Error on process message, let\'s retry', JSON.stringify(content));
     return ConsumerResult.Retry;
   }
 
+
   return ConsumerResult.Ok;
 });
 
-const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '2'}});
+const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '5000'}});
 console.log('Sending message');
 
 for (let i = 0; i < 5; i++) {
@@ -61,9 +65,9 @@ for (let i = 0; i < 5; i++) {
         headers: {key: Buffer.from('value1')},
       },
     );
-    console.log(result);
+    console.log("Js message sent. Offset: ", result);
     await timersPromises.setTimeout(1200);
   } catch (error) {
-    console.error('Error on send', error);
+    console.error('Js Error on send', error);
   }
 }

package/index.d.ts

@@ -9,6 +9,13 @@ export const enum SecurityProtocol {
   SaslPlaintext = 'SaslPlaintext',
   SaslSsl = 'SaslSsl'
 }
+export interface KafkaConfiguration {
+  brokers: string
+  clientId: string
+  securityProtocol?: SecurityProtocol
+  configuration?: Record<string, string>
+  enableAnsiLogger?: boolean
+}
 export interface ConsumerConfiguration {
   topic: string
   groupId: string
@@ -70,20 +77,14 @@ export const enum ConsumerResult {
   Ok = 'Ok',
   Retry = 'Retry'
 }
-export class KafkaConfiguration {
-  brokers: string
-  clientId: string
-  securityProtocol?: SecurityProtocol
-  configuration?: Record<string, string>
-}
 export class KafkaClient {
   readonly kafkaConfiguration: KafkaConfiguration
-  constructor(brokers: string, clientId: string, securityProtocol?: SecurityProtocol | undefined | null, configuration?: Record<string, string> | undefined | null)
+  constructor(kafkaConfiguration: KafkaConfiguration)
   createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
   createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
 }
 export class KafkaConsumer {
-  startConsumer(callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult>): Promise<void>
+  startConsumer(callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult | undefined>): Promise<void>
 }
 export class KafkaProducer {
   send(message: MessageModel): Promise<OwnedDelivery>

package/index.js

@@ -252,10 +252,9 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SecurityProtocol, KafkaConfiguration, KafkaClient, KafkaConsumer, KafkaProducer, AutoOffsetReset, KafkaCommitMode, PartitionPosition, ConsumerResult } = nativeBinding
+const { SecurityProtocol, KafkaClient, KafkaConsumer, KafkaProducer, AutoOffsetReset, KafkaCommitMode, PartitionPosition, ConsumerResult } = nativeBinding
 
 module.exports.SecurityProtocol = SecurityProtocol
-module.exports.KafkaConfiguration = KafkaConfiguration
 module.exports.KafkaClient = KafkaClient
 module.exports.KafkaConsumer = KafkaConsumer
 module.exports.KafkaProducer = KafkaProducer

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.0.3",
+  "version": "0.0.9",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -15,6 +15,15 @@
     }
   },
   "license": "MIT",
+  "keywords": [
+    "kafka",
+    "napi-rs",
+    "librdkafka",
+    "typescript",
+    "rust"
+  ],
+  "author": "inaiat@gmail.com",
+  "homepage": "https://github.com/inaiat/kafka-crab-js.git",
   "devDependencies": {
     "@napi-rs/cli": "^2.16.1",
     "ava": "^5.3.1",
@@ -41,8 +50,8 @@
   },
   "packageManager": "yarn@3.6.0",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.0.3",
-    "kafka-crab-js-darwin-x64": "0.0.3",
-    "kafka-crab-js-linux-x64-gnu": "0.0.3"
+    "kafka-crab-js-darwin-arm64": "0.0.9",
+    "kafka-crab-js-darwin-x64": "0.0.9",
+    "kafka-crab-js-linux-x64-gnu": "0.0.9"
   }
 }

package/src/kafka/kafka_client.rs

@@ -55,12 +55,13 @@ impl fmt::Display for SecurityProtocol {
 }
 
 #[derive(Clone, Debug)]
-#[napi]
+#[napi(object)]
 pub struct KafkaConfiguration {
   pub brokers: String,
   pub client_id: String,
   pub security_protocol: Option<SecurityProtocol>,
   pub configuration: Option<HashMap<String, String>>,
+  pub enable_ansi_logger: Option<bool>,
 }
 
 #[derive(Clone, Debug)]
@@ -74,23 +75,18 @@ pub struct KafkaClient {
 #[napi]
 impl KafkaClient {
   #[napi(constructor)]
-  pub fn new(
-    brokers: String,
-    client_id: String,
-    security_protocol: Option<SecurityProtocol>,
-    configuration: Option<HashMap<String, String>>,
-  ) -> Self {
-    match tracing_subscriber::fmt::try_init() {
+  pub fn new(kafka_configuration: KafkaConfiguration) -> Self {
+    match tracing_subscriber::fmt()
+      .with_ansi(kafka_configuration.enable_ansi_logger.unwrap_or(false))
+      .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+      .try_init()
+    {
       Ok(_) => {}
-      Err(e) => println!("Failed to initialize tracing: {:?}", e),
-    }
-
-    KafkaClient::with_kafka_configuration(KafkaConfiguration {
-      brokers,
-      client_id,
-      security_protocol,
-      configuration,
-    })
+      Err(e) => {
+        eprintln!("Failed to initialize tracing: {:?}", e);
+      }
+    };
+    KafkaClient::with_kafka_configuration(kafka_configuration)
   }
 
   pub fn get_client_config(&self) -> &ClientConfig {
@@ -103,11 +99,12 @@ impl KafkaClient {
       security_protocol,
       client_id,
       configuration,
+      ..
     } = kafka_configuration.clone();
 
     let mut rdkafka_client_config = ClientConfig::new();
 
-    rdkafka_client_config.set_log_level(RDKafkaLogLevel::Info);
+    rdkafka_client_config.set_log_level(RDKafkaLogLevel::Debug);
     rdkafka_client_config.set("bootstrap.servers", brokers);
     rdkafka_client_config.set("client.id", client_id);
     if let Some(security_protocol) = security_protocol {

package/src/kafka/kafka_consumer.rs

@@ -92,7 +92,9 @@ impl KafkaConsumer {
     })
   }
 
-  #[napi(ts_args_type = "callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult>")]
+  #[napi(
+    ts_args_type = "callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult | undefined>"
+  )]
   pub async fn start_consumer(&self, func: ThreadsafeFunction<Buffer>) -> Result<()> {
     let ConsumerConfiguration {
       commit_mode,
@@ -304,22 +306,22 @@ async fn process_message(
   match message.payload_view::<[u8]>() {
     None => Ok(None),
     Some(Ok(payload)) => match func
-      .call_async::<Promise<ConsumerResult>>(Ok(payload.into()))
+      .call_async::<Promise<Option<ConsumerResult>>>(Ok(payload.into()))
       .await
     {
       Ok(js_result) => match js_result.await {
         Ok(value) => match value {
-          ConsumerResult::Ok => {
+          None | Some(ConsumerResult::Ok) => {
             if let Some(commit_mode) = commit_mode {
               match stream_consumer.commit_message(message, *commit_mode) {
-                Ok(_) => Ok(Some(value)),
+                Ok(_) => Ok(Some(ConsumerResult::Ok)),
                 Err(e) => Err(anyhow::Error::new(e)),
               }
             } else {
-              Ok(Some(value))
+              Ok(Some(ConsumerResult::Ok))
             }
           }
-          ConsumerResult::Retry => Ok(Some(value)),
+          Some(ConsumerResult::Retry) => Ok(Some(ConsumerResult::Retry)),
         },
         Err(e) => Err(anyhow::Error::new(e)),
       },