kafka-crab-js 0.0.3 → 0.0.10

package/Cargo.toml

@@ -8,16 +8,24 @@ crate-type = ["cdylib"]
 
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2.13", default-features = false, features = ["napi4", "tokio_rt"] }
+napi = { version = "2.13", default-features = false, features = [
+    "napi4",
+    "tokio_rt",
+] }
 napi-derive = "2.13"
 
-anyhow = { version= "1.0", features = ["backtrace"]}
+anyhow = { version = "1.0", features = ["backtrace"] }
 
-rdkafka = { version = "0.32", features = ["libz-static", "cmake-build", "tokio", "ssl-vendored"] }
+rdkafka = { version = "0.32", features = [
+    "libz-static",
+    "cmake-build",
+    "tokio",
+    "ssl-vendored",
+] }
 tokio = { version = "1", features = ["full"] }
 
-tracing            = "0.1"
-tracing-subscriber = "0.3"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 
 [build-dependencies]
 napi-build = "2.0.1"

package/README.md

@@ -1,16 +1,77 @@
-Kafka Crab Js
--------------
+# Kafka Crab Js
 
 **Blazingly Fast Kafka Client for NodeJs**
 
 Kafka-Crab-JS is a powerful Node.js library that allows developers to interact with Apache Kafka using Rust programming language, seamlessly integrated with Node.js applications. This documentation provides an overview of the Kafka-Crab-JS library, including installation instructions, usage guidelines, and examples.
 
+The plugin can be found on [npm](https://www.npmjs.com/package/kafka-crab-js) and the source code is available on [GitHub](https://github.com/inaiat/kafka-crab-js).
 
-# Installation
+## Table of Contents
+1. [Installation](#installation)
+2. [Usage](#usage)
+   - [Client](#client)
+   - [Producer](#producer)
+   - [Consumer](#consumer)
+3. [Examples](#examples)
 
-To use Kafka-Crab-JS in your Node.js project, follow the steps below:
 
-todo
+## Installation
+To use Kafka Crab JS, you need to have Node.js and npm (Node Package Manager) installed on your system. Once you have them set up, you can install the plugin by running the following command:
 
-# Examples
-todo
+```shell
+npm install kafka-crab-js
+```
+## Usage
+### Client
+Creating a Client Instance
+Next, you can create a client instance by specifying the Kafka broker(s) and other optional configurations:
+
+```javascript
+import { KafkaClient }
+
+const kafkaClient = new KafkaClient({
+  brokers: 'localhost:29092',
+  clientId: 'my-id'});
+```
+### Producer
+The producer component allows you to send messages to Kafka topics.
+
+```javascript
+const result = await producer.send({
+    topic: 'my-js-topic',
+    key: Buffer.from('abc'),
+    value: Buffer.from(`{"name":"Elizeu Drummond","phone":"55219123456"}`),
+    headers: {key: Buffer.from('value1')},
+});
+console.log('Message sent with offset:', result);
+```
+
+
+### Consumer
+
+**Creating a Consumer Instance**
+
+Create a consumer instance by specifying the Kafka broker(s), group ID, and other optional configurations:
+
+```javascript
+const consumer = kafkaClient.createConsumer({
+  topic: 'my-js-topic',
+  groupId: 'my-js-group',
+  retryStrategy: {
+    retries: 3,
+  },
+});
+```
+
+You can consume messages using the run startConsumer:
+
+```javascript
+consumer.startConsumer(async (error, value) => {
+  const message = JSON.parse(value.toString());
+  const content = await process_my_message(message);
+  return ConsumerResult.Ok;
+});
+```
+## Examples
+
+You can find examples of using Kafka Crab JS in the [GitHub repository](https://github.com/inaiat/kafka-crab-js).

package/example/sample.mjs

@@ -3,14 +3,17 @@ import timersPromises from 'node:timers/promises';
 import {Buffer} from 'node:buffer';
 import {ConsumerResult, KafkaClient, KafkaCommitMode, PartitionPosition} from '../index.js';
 
-const kafkaClient = new KafkaClient('localhost:29092', 'my-id');
+const kafkaClient = new KafkaClient({ 
+  brokers: 'localhost:29092', 
+  clientId: 'my-id', 
+  enableAnsiLogger: false});
 const topic = 'my-js-topic';
 
 let counter = 0;
 
 async function process(message) {
   await timersPromises.setTimeout(1);
-  return 'message received: ' + message;
+  return 'message received: ' + JSON.stringify(message);
 }
 
 const consumer = kafkaClient.createConsumer({
@@ -27,7 +30,7 @@ console.time('consumer');
 
 consumer.startConsumer(async (error, value) => {
   if (error) {
-    console.error('Consumer error', error);
+    console.error('Js Consumer error', error);
     return;
   }
 
@@ -36,19 +39,20 @@ consumer.startConsumer(async (error, value) => {
   counter++;
   if (counter >= 0) {
     console.timeEnd('consumer');
-    console.log('Counter:', counter, 'Content:', JSON.stringify(content));
+    console.log('Js Counter:', counter, 'Content:', JSON.stringify(content));
     console.time('consumer');
   }
 
   if (message._id === '3') {
-    console.error('Error on process message, let\'s retry', message);
+    console.error('Js Error on process message, let\'s retry', JSON.stringify(content));
     return ConsumerResult.Retry;
   }
 
+
   return ConsumerResult.Ok;
 });
 
-const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '2'}});
+const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '5000'}});
 console.log('Sending message');
 
 for (let i = 0; i < 5; i++) {
@@ -61,9 +65,9 @@ for (let i = 0; i < 5; i++) {
         headers: {key: Buffer.from('value1')},
       },
     );
-    console.log(result);
+    console.log("Js message sent. Offset: ", result);
     await timersPromises.setTimeout(1200);
   } catch (error) {
-    console.error('Error on send', error);
+    console.error('Js Error on send', error);
   }
 }

package/index.d.ts

@@ -9,6 +9,13 @@ export const enum SecurityProtocol {
   SaslPlaintext = 'SaslPlaintext',
   SaslSsl = 'SaslSsl'
 }
+export interface KafkaConfiguration {
+  brokers: string
+  clientId: string
+  securityProtocol?: SecurityProtocol
+  configuration?: Record<string, string>
+  enableAnsiLogger?: boolean
+}
 export interface ConsumerConfiguration {
   topic: string
   groupId: string
@@ -70,20 +77,14 @@ export const enum ConsumerResult {
   Ok = 'Ok',
   Retry = 'Retry'
 }
-export class KafkaConfiguration {
-  brokers: string
-  clientId: string
-  securityProtocol?: SecurityProtocol
-  configuration?: Record<string, string>
-}
 export class KafkaClient {
   readonly kafkaConfiguration: KafkaConfiguration
-  constructor(brokers: string, clientId: string, securityProtocol?: SecurityProtocol | undefined | null, configuration?: Record<string, string> | undefined | null)
+  constructor(kafkaConfiguration: KafkaConfiguration)
   createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
   createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
 }
 export class KafkaConsumer {
-  startConsumer(callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult>): Promise<void>
+  startConsumer(callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult | undefined>): Promise<void>
 }
 export class KafkaProducer {
   send(message: MessageModel): Promise<OwnedDelivery>

package/index.js

@@ -252,10 +252,9 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SecurityProtocol, KafkaConfiguration, KafkaClient, KafkaConsumer, KafkaProducer, AutoOffsetReset, KafkaCommitMode, PartitionPosition, ConsumerResult } = nativeBinding
+const { SecurityProtocol, KafkaClient, KafkaConsumer, KafkaProducer, AutoOffsetReset, KafkaCommitMode, PartitionPosition, ConsumerResult } = nativeBinding
 
 module.exports.SecurityProtocol = SecurityProtocol
-module.exports.KafkaConfiguration = KafkaConfiguration
 module.exports.KafkaClient = KafkaClient
 module.exports.KafkaConsumer = KafkaConsumer
 module.exports.KafkaProducer = KafkaProducer

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.0.3",
+  "version": "0.0.10",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -15,6 +15,15 @@
     }
   },
   "license": "MIT",
+  "keywords": [
+    "kafka",
+    "napi-rs",
+    "librdkafka",
+    "typescript",
+    "rust"
+  ],
+  "author": "inaiat@gmail.com",
+  "homepage": "https://github.com/inaiat/kafka-crab-js.git",
   "devDependencies": {
     "@napi-rs/cli": "^2.16.1",
     "ava": "^5.3.1",
@@ -41,8 +50,8 @@
   },
   "packageManager": "yarn@3.6.0",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.0.3",
-    "kafka-crab-js-darwin-x64": "0.0.3",
-    "kafka-crab-js-linux-x64-gnu": "0.0.3"
+    "kafka-crab-js-darwin-arm64": "0.0.10",
+    "kafka-crab-js-darwin-x64": "0.0.10",
+    "kafka-crab-js-linux-x64-gnu": "0.0.10"
   }
 }

package/src/kafka/kafka_client.rs

@@ -55,12 +55,13 @@ impl fmt::Display for SecurityProtocol {
 }
 
 #[derive(Clone, Debug)]
-#[napi]
+#[napi(object)]
 pub struct KafkaConfiguration {
   pub brokers: String,
   pub client_id: String,
   pub security_protocol: Option<SecurityProtocol>,
   pub configuration: Option<HashMap<String, String>>,
+  pub enable_ansi_logger: Option<bool>,
 }
 
 #[derive(Clone, Debug)]
@@ -74,23 +75,18 @@ pub struct KafkaClient {
 #[napi]
 impl KafkaClient {
   #[napi(constructor)]
-  pub fn new(
-    brokers: String,
-    client_id: String,
-    security_protocol: Option<SecurityProtocol>,
-    configuration: Option<HashMap<String, String>>,
-  ) -> Self {
-    match tracing_subscriber::fmt::try_init() {
+  pub fn new(kafka_configuration: KafkaConfiguration) -> Self {
+    match tracing_subscriber::fmt()
+      .with_ansi(kafka_configuration.enable_ansi_logger.unwrap_or(false))
+      .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+      .try_init()
+    {
       Ok(_) => {}
-      Err(e) => println!("Failed to initialize tracing: {:?}", e),
-    }
-
-    KafkaClient::with_kafka_configuration(KafkaConfiguration {
-      brokers,
-      client_id,
-      security_protocol,
-      configuration,
-    })
+      Err(e) => {
+        eprintln!("Failed to initialize tracing: {:?}", e);
+      }
+    };
+    KafkaClient::with_kafka_configuration(kafka_configuration)
   }
 
   pub fn get_client_config(&self) -> &ClientConfig {
@@ -103,11 +99,12 @@ impl KafkaClient {
       security_protocol,
       client_id,
       configuration,
+      ..
     } = kafka_configuration.clone();
 
     let mut rdkafka_client_config = ClientConfig::new();
 
-    rdkafka_client_config.set_log_level(RDKafkaLogLevel::Info);
+    rdkafka_client_config.set_log_level(RDKafkaLogLevel::Debug);
     rdkafka_client_config.set("bootstrap.servers", brokers);
     rdkafka_client_config.set("client.id", client_id);
     if let Some(security_protocol) = security_protocol {

package/src/kafka/kafka_consumer.rs

@@ -92,7 +92,9 @@ impl KafkaConsumer {
     })
   }
 
-  #[napi(ts_args_type = "callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult>")]
+  #[napi(
+    ts_args_type = "callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult | undefined>"
+  )]
   pub async fn start_consumer(&self, func: ThreadsafeFunction<Buffer>) -> Result<()> {
     let ConsumerConfiguration {
       commit_mode,
@@ -304,22 +306,22 @@ async fn process_message(
   match message.payload_view::<[u8]>() {
     None => Ok(None),
     Some(Ok(payload)) => match func
-      .call_async::<Promise<ConsumerResult>>(Ok(payload.into()))
+      .call_async::<Promise<Option<ConsumerResult>>>(Ok(payload.into()))
       .await
     {
       Ok(js_result) => match js_result.await {
         Ok(value) => match value {
-          ConsumerResult::Ok => {
+          None | Some(ConsumerResult::Ok) => {
             if let Some(commit_mode) = commit_mode {
               match stream_consumer.commit_message(message, *commit_mode) {
-                Ok(_) => Ok(Some(value)),
+                Ok(_) => Ok(Some(ConsumerResult::Ok)),
                 Err(e) => Err(anyhow::Error::new(e)),
               }
             } else {
-              Ok(Some(value))
+              Ok(Some(ConsumerResult::Ok))
             }
           }
-          ConsumerResult::Retry => Ok(Some(value)),
+          Some(ConsumerResult::Retry) => Ok(Some(ConsumerResult::Retry)),
         },
         Err(e) => Err(anyhow::Error::new(e)),
       },