kafka-crab-js 0.4.1 → 0.6.0

package/Cargo.toml

@@ -8,21 +8,20 @@ crate-type = ["cdylib"]
 
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2", default-features = false, features = [
+napi = { version = "2.16", default-features = false, features = [
     "napi8",
     "tokio_rt",
     "serde-json",
 ] }
-napi-derive = "2"
+napi-derive = "2.16"
 
 anyhow = { version = "1.0", features = ["backtrace"] }
 
-rdkafka = { version = "0.34", features = [
+rdkafka = { version = "0.36", features = [
     "libz-static",
     "cmake-build",
     "tokio",
     "ssl-vendored",
-    "ssl",
 ] }
 tokio = { version = "1", features = ["full"] }
 
@@ -34,3 +33,4 @@ napi-build = "2.0.1"
 
 [profile.release]
 lto = true
+strip = "symbols"

package/example/sample.mjs

@@ -1,14 +1,17 @@
 /* eslint-disable no-await-in-loop */
 
-import timersPromises from 'node:timers/promises';
+// import timersPromises from 'node:timers/promises';
 import { Buffer } from 'node:buffer';
 import { nanoid } from 'nanoid';
-import { KafkaClient, CommitMode, PartitionPosition, ConsumerResult } from '../index.js';
+import {
+  KafkaClient, CommitMode, PartitionPosition, ConsumerResult,
+} from '../index.js';
 
 const kafkaClient = new KafkaClient({
   brokers: 'localhost:29092',
   clientId: 'my-js-group',
   logLevel: 'info',
+  brokerAddressFamily: 'v4',
 });
 const topic = `topic-${nanoid()}`;
 
@@ -31,7 +34,7 @@ async function produce() {
       const result = await producer.send(
         {
           topic,
-          messages: [{ value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`) }],
+          messages: [{ payload: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`) }],
         },
       );
       console.log('Js message sent. Offset:', result);

package/example/stream_consumer.mjs

@@ -0,0 +1,99 @@
+import { CommitMode, KafkaClient, KafkaStreamConsumer, PartitionPosition } from "../index.js";
+import { setTimeout as sleep } from 'node:timers/promises';
+import { fakerPT_BR  } from '@faker-js/faker';
+import { Readable } from "node:stream";
+
+const kafkaClient = new KafkaClient({
+    brokers: 'localhost:29092',
+    clientId: 'my-js-group-11',
+    logLevel: 'debug',
+    brokerAddressFamily: 'v4',
+  });
+
+/**
+ * Produces some messages to the topic
+ * @param {string} topic 
+ * @param {number} messages
+ */
+async function produce(topic, messages=1) {
+  const producer = kafkaClient.createProducer({ topic, configuration: { 'message.timeout.ms': '5000' } });
+  const index = Math.floor(Math.random() * 100_000)
+  for (let i = index ?? 0; i < index+messages; i++) {
+    try {
+      const result = await producer.send(
+        {
+          topic,
+          messages: [{ value: Buffer.from(`{"_id":"${i}","name":"${fakerPT_BR.person.fullName()}","phone":"${fakerPT_BR.phone.number()}"}`) }],
+        },
+      );
+      console.log('Js message sent. Offset:', result);
+    } catch (error) {
+      console.error('Js Error on send', error);
+    }
+  }
+}
+
+class KafkaStreamReadable extends Readable {
+  /**
+   * Creates a new KafkaStream
+   * @param {KafkaStreamConsumer} kafkaStreamConsumer 
+   */
+  constructor(kafkaStreamConsumer) {
+    super({ objectMode: true });
+    this.kafkaStreamConsumer = kafkaStreamConsumer;
+  }
+
+  async _read() {
+    let message = await this.kafkaStreamConsumer.recv();
+    this.push(message)
+  }
+}
+
+async function startConsumer() {
+    const kafkaStreamConsumer = kafkaClient.createStreamConsumer({
+      groupId: `crab-test`,
+      enableAutoCommit: true,
+    });
+  
+    // await streamerConsumer.subscribe(
+    //   [{ 
+    //     topic: 'foo', 
+    //     partitionOffset: [
+    //       { partition:0, offset: { position: PartitionPosition.Beginning }} 
+    //     ]
+    //   }, 
+    //   { 
+    //     topic: 'bar', 
+    //     partitionOffset: [
+    //       { partition:0, offset: { position: PartitionPosition.Stored }},
+    //       { partition:1, offset: { position: PartitionPosition.Beginning }} 
+    //     ]
+    //   }]
+    // );
+    // await streamerConsumer.subscribe([ { topic: 'foo', allOffsets: { position: PartitionPosition.Stored } } ]);
+    // await streamerConsumer.subscribe('foo');
+    await kafkaStreamConsumer.subscribe([ { topic: 'foo' }, { topic: 'bar' } ]);
+    
+    const maxMessages = 20;
+    let counter = 0;
+    const kafkaStream = new KafkaStreamReadable(kafkaStreamConsumer);
+    console.log('Starting consumer')
+    kafkaStream.on('data', (message) => {
+      counter++;
+      console.log('>>> Message received:', { payload: message.payload.toString(), offset: message.offset, partition: message.partition, topic: message.topic })
+      if (counter === maxMessages) {
+        kafkaStreamConsumer.unsubscribe();
+        kafkaStream.destroy();
+      }
+      // streamerConsumer.commit(message.partition, message.offset+1, CommitMode.Sync)
+    })
+  }
+
+  if (process.argv[2] === 'send') {
+    const topic = process.argv[3]
+    const messages = process.argv[4] ? Number(process.argv[4]) : 1
+    console.log('Sending', messages, 'messages to', topic)
+    await produce(topic, messages);
+  } else { 
+    await startConsumer();
+  }

package/index.d.ts

@@ -3,10 +3,20 @@
 
 /* auto-generated by NAPI-RS */
 
-export const enum CommitMode {
-  AutoCommit = 'AutoCommit',
-  Sync = 'Sync',
-  Async = 'Async'
+export const enum ConsumerResult {
+  Ok = 'Ok',
+  Retry = 'Retry'
+}
+export interface KafkaConsumerConfiguration {
+  topic: string
+  groupId: string
+  retryStrategy?: RetryStrategy
+  offset?: OffsetModel
+  createTopic?: boolean
+  commitMode?: CommitMode
+  enableAutoCommit?: boolean
+  configuration?: Record<string, string>
+  fecthMetadataTimeout?: number
 }
 export interface RetryStrategy {
   retries: number
@@ -16,19 +26,16 @@ export interface RetryStrategy {
   offset?: OffsetModel
   configuration?: Record<string, string>
 }
-export const enum ConsumerResult {
-  Ok = 'Ok',
-  Retry = 'Retry'
+export const enum CommitMode {
+  Sync = 'Sync',
+  Async = 'Async'
 }
 export interface ConsumerConfiguration {
-  topic: string
   groupId: string
-  retryStrategy?: RetryStrategy
-  offset?: OffsetModel
   createTopic?: boolean
-  commitMode?: CommitMode
   enableAutoCommit?: boolean
   configuration?: Record<string, string>
+  fecthMetadataTimeout?: number
 }
 export const enum SecurityProtocol {
   Plaintext = 'Plaintext',
@@ -42,15 +49,7 @@ export interface KafkaConfiguration {
   securityProtocol?: SecurityProtocol
   configuration?: Record<string, string>
   logLevel?: string
-}
-export interface ProducerConfiguration {
-  queueTimeout?: number
-  thrownOnError?: boolean
-  configuration?: Record<string, string>
-}
-export interface ProducerRecord {
-  topic: string
-  messages: Array<MessageModel>
+  brokerAddressFamily?: string
 }
 export const enum AutoOffsetReset {
   Smallest = 'Smallest',
@@ -61,26 +60,23 @@ export const enum AutoOffsetReset {
   End = 'End',
   Error = 'Error'
 }
-export interface Payload {
-  value: Buffer
-  key?: Buffer
-  headers?: Record<string, Buffer>
-  topic: string
-  partition: number
-  offset: number
-}
 export const enum PartitionPosition {
-  Beginning = 0,
-  End = 1,
-  Stored = 2
+  Beginning = 'Beginning',
+  End = 'End',
+  Stored = 'Stored'
 }
 export interface OffsetModel {
   offset?: number
   position?: PartitionPosition
 }
-export interface KafkaCrabError {
-  code: number
-  message: string
+export interface PartitionOffset {
+  partition: number
+  offset: OffsetModel
+}
+export interface TopicPartitionConfig {
+  topic: string
+  allOffsets?: OffsetModel
+  partitionOffset?: Array<PartitionOffset>
 }
 export interface RecordMetadata {
   topic: string
@@ -88,19 +84,56 @@ export interface RecordMetadata {
   offset: number
   error?: KafkaCrabError
 }
-export interface MessageModel {
+export interface MessageProducer {
+  value: Buffer
+  key?: Buffer
+  headers?: Record<string, Buffer>
+}
+export interface ProducerConfiguration {
+  queueTimeout?: number
+  thrownOnError?: boolean
+  configuration?: Record<string, string>
+}
+export interface ProducerRecord {
+  topic: string
+  messages: Array<MessageProducer>
+}
+export interface Payload {
   value: Buffer
   key?: Buffer
   headers?: Record<string, Buffer>
+  topic: string
+  partition: number
+  offset: number
+}
+export interface Message {
+  payload: Buffer
+  key?: Buffer
+  headers?: Record<string, Buffer>
+  topic: string
+  partition: number
+  offset: number
+}
+export interface KafkaCrabError {
+  code: number
+  message: string
 }
 export class KafkaConsumer {
   startConsumer(callback: (error: Error | undefined, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
 }
+export class KafkaStreamConsumer {
+  subscribe(topicConfigs: string | Array<TopicPartitionConfig>): Promise<void>
+  unsubscribe(): void
+  seek(topic: string, partition: number, offsetModel: OffsetModel, timeout?: number | undefined | null): void
+  recv(): Promise<Message>
+  commit(topic: string, partition: number, offset: number, commit: CommitMode): void
+}
 export class KafkaClient {
   readonly kafkaConfiguration: KafkaConfiguration
   constructor(kafkaConfiguration: KafkaConfiguration)
   createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
-  createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
+  createConsumer(consumerConfiguration: KafkaConsumerConfiguration): KafkaConsumer
+  createStreamConsumer(consumerConfiguration: ConsumerConfiguration): KafkaStreamConsumer
 }
 export class KafkaProducer {
   send(producerRecord: ProducerRecord): Promise<Array<RecordMetadata>>

package/index.js

@@ -224,14 +224,72 @@ switch (platform) {
         }
         break
       case 'arm':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-arm-musleabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-arm-musleabihf.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-arm-musleabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-arm-gnueabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-arm-gnueabihf.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-arm-gnueabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'riscv64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-riscv64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-riscv64-musl.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-riscv64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-riscv64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-riscv64-gnu.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-riscv64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 's390x':
         localFileExisted = existsSync(
-          join(__dirname, 'kafka-crab-js.linux-arm-gnueabihf.node')
+          join(__dirname, 'kafka-crab-js.linux-s390x-gnu.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./kafka-crab-js.linux-arm-gnueabihf.node')
+            nativeBinding = require('./kafka-crab-js.linux-s390x-gnu.node')
           } else {
-            nativeBinding = require('kafka-crab-js-linux-arm-gnueabihf')
+            nativeBinding = require('kafka-crab-js-linux-s390x-gnu')
           }
         } catch (e) {
           loadError = e
@@ -252,13 +310,14 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { CommitMode, ConsumerResult, KafkaConsumer, SecurityProtocol, KafkaClient, KafkaProducer, AutoOffsetReset, PartitionPosition } = nativeBinding
+const { ConsumerResult, KafkaConsumer, KafkaStreamConsumer, CommitMode, SecurityProtocol, KafkaClient, AutoOffsetReset, PartitionPosition, KafkaProducer } = nativeBinding
 
-module.exports.CommitMode = CommitMode
 module.exports.ConsumerResult = ConsumerResult
 module.exports.KafkaConsumer = KafkaConsumer
+module.exports.KafkaStreamConsumer = KafkaStreamConsumer
+module.exports.CommitMode = CommitMode
 module.exports.SecurityProtocol = SecurityProtocol
 module.exports.KafkaClient = KafkaClient
-module.exports.KafkaProducer = KafkaProducer
 module.exports.AutoOffsetReset = AutoOffsetReset
 module.exports.PartitionPosition = PartitionPosition
+module.exports.KafkaProducer = KafkaProducer

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.4.1",
+  "version": "0.6.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -24,19 +24,24 @@
   ],
   "author": "inaiat@gmail.com",
   "homepage": "https://github.com/inaiat/kafka-crab-js.git",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/inaiat/kafka-crab-js.git"
+  },
   "devDependencies": {
-    "@napi-rs/cli": "^2.16.3",
-    "ava": "^5.3.1",
-    "benny": "^3.7.1",
-    "kafkajs": "^2.2.4",
-    "nanoid": "^4.0.2",
-    "xo": "^0.54.2"
+    "@faker-js/faker": "^8.4.1",
+    "@napi-rs/cli": "^2.18.3",
+    "@types/node": "^20.12.12",
+    "ava": "^6.1.2",
+    "nanoid": "^5.0.7",
+    "typescript": "^5.4.5",
+    "xo": "^0.58.0"
   },
   "ava": {
     "timeout": "3m"
   },
   "engines": {
-    "node": ">= 10"
+    "node": ">= 18"
   },
   "xo": {
     "space": true,
@@ -54,10 +59,10 @@
     "universal": "napi universal",
     "version": "napi version"
   },
-  "packageManager": "yarn@3.6.0",
+  "packageManager": "pnpm@9.1.1",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.4.1",
-    "kafka-crab-js-darwin-x64": "0.4.1",
-    "kafka-crab-js-linux-x64-gnu": "0.4.1"
+    "kafka-crab-js-darwin-arm64": "0.6.0",
+    "kafka-crab-js-darwin-x64": "0.6.0",
+    "kafka-crab-js-linux-x64-gnu": "0.6.0"
   }
 }

package/src/kafka/consumer/consumer_helper.rs

@@ -0,0 +1,179 @@
+use std::{collections::HashMap, time::Duration};
+
+use rdkafka::{
+  config::RDKafkaLogLevel,
+  consumer::{Consumer, StreamConsumer},
+  ClientConfig, TopicPartitionList,
+};
+use tracing::{debug, error, info, warn};
+
+use crate::kafka::{
+  consumer::model::LoggingConsumer,
+  kafka_admin::KafkaAdmin,
+  kafka_util::convert_to_rdkafka_offset,
+  model::{OffsetModel, PartitionOffset},
+};
+
+use super::{model::ConsumerConfiguration, model::CustomContext};
+
+pub async fn create_stream_consumer_and_setup_everything(
+  client_config: &ClientConfig,
+  consumer_configuration: &ConsumerConfiguration,
+  topic: &str,
+  offset: &Option<OffsetModel>,
+  configuration: Option<HashMap<String, String>>,
+  timeout: Duration,
+) -> anyhow::Result<StreamConsumer<CustomContext>> {
+  let consumer = create_stream_consumer(client_config, consumer_configuration, configuration)?;
+
+  if consumer_configuration.create_topic.unwrap_or(true) {
+    try_create_topic(topic, client_config).await?;
+  }
+
+  if let Some(offset_model) = offset {
+    set_offset_of_all_partitions(offset_model, &consumer, topic, timeout)?;
+  }
+
+  try_subscribe(&consumer, topic)?;
+
+  Ok(consumer)
+}
+
+pub fn create_stream_consumer(
+  client_config: &ClientConfig,
+  consumer_configuration: &ConsumerConfiguration,
+  configuration: Option<HashMap<String, String>>,
+) -> anyhow::Result<StreamConsumer<CustomContext>> {
+  let context = CustomContext;
+
+  let ConsumerConfiguration {
+    group_id,
+    enable_auto_commit,
+    ..
+  } = consumer_configuration.clone();
+
+  let mut consumer_config: ClientConfig = client_config.clone();
+
+  if let Some(config) = configuration {
+    consumer_config.extend(config);
+  }
+
+  debug!(
+    "Creating consumer with configuration: {:?}",
+    consumer_config
+  );
+
+  let consumer: LoggingConsumer = consumer_config
+    .clone()
+    .set("group.id", group_id.clone())
+    .set(
+      "enable.auto.commit",
+      enable_auto_commit.unwrap_or(true).to_string(),
+    )
+    .set_log_level(RDKafkaLogLevel::Debug)
+    .create_with_context(context)?;
+
+  info!("Consumer created. Group id: {:?}", group_id);
+  Ok(consumer)
+}
+
+pub fn try_subscribe(consumer: &LoggingConsumer, topic: &str) -> anyhow::Result<()> {
+  consumer
+    .subscribe(vec![&*topic.to_string()].as_slice())
+    .map_err(|e| {
+      anyhow::Error::msg(format!(
+        "Can't subscribe to specified topics. Error: {:?}",
+        e
+      ))
+    })?;
+  debug!("Subscribed to topic: {:?}", topic);
+  Ok(())
+}
+
+pub async fn try_create_topic(topic: &str, client_config: &ClientConfig) -> anyhow::Result<()> {
+  info!("Creating topic: {:?}", topic);
+  let admin = KafkaAdmin::new(client_config, None);
+  let result = admin.create_topic(topic).await;
+  if let Err(e) = result {
+    warn!("Fail to create topic {:?}", e);
+    return Err(anyhow::Error::msg(format!("Fail to create topic: {:?}", e)));
+  }
+  info!("Topic created: {:?}", topic);
+  Ok(())
+}
+
+pub fn set_offset_of_all_partitions(
+  offset_model: &OffsetModel,
+  consumer: &StreamConsumer<CustomContext>,
+  topic: &str,
+  timeout: Duration,
+) -> anyhow::Result<()> {
+  let offset = convert_to_rdkafka_offset(offset_model);
+  debug!("Setting offset to: {:?}", offset);
+  let metadata = consumer.fetch_metadata(Some(topic), timeout)?;
+
+  metadata.topics().iter().for_each(|meta_topic| {
+    let mut tpl = TopicPartitionList::new();
+    meta_topic.partitions().iter().for_each(|meta_partition| {
+      info!("Adding partition: {:?}", meta_partition.id());
+      tpl.add_partition(topic, meta_partition.id());
+    });
+    match tpl.set_all_offsets(offset) {
+      Ok(_) => {
+        debug!("Offset set to: {:?}", offset);
+      }
+      Err(e) => {
+        error!("Fail to set offset: {:?}", e)
+      }
+    };
+    match consumer.assign(&tpl) {
+      Ok(_) => {
+        debug!("Assigning topic: {:?}", topic);
+      }
+      Err(e) => {
+        error!("Fail to assign topic: {:?}", e);
+      }
+    }
+  });
+
+  Ok(())
+}
+
+pub fn assign_offset_or_use_metadata(
+  topic: &str,
+  partition_offset: Option<Vec<PartitionOffset>>,
+  offset_model: Option<&OffsetModel>,
+  consumer: &StreamConsumer<CustomContext>,
+  timeout: Duration,
+) -> anyhow::Result<()> {
+  let mut tpl = TopicPartitionList::new();
+
+  if let Some(value) = partition_offset {
+    for item in value {
+      let offset = convert_to_rdkafka_offset(&item.offset);
+      debug!(
+        "Adding partition: {:?} with offset: {:?} for topic: {:?}",
+        item.partition, offset, topic
+      );
+      tpl.add_partition_offset(topic, item.partition, offset)?;
+    }
+  } else if let Some(offset_model) = offset_model {
+    let offset = convert_to_rdkafka_offset(offset_model);
+    let metadata = consumer.fetch_metadata(Some(topic), timeout)?;
+    for meta_topic in metadata.topics() {
+      for meta_partition in meta_topic.partitions() {
+        info!(
+          "Adding partition: {:?} with offset: {:?} for topic: {:?}",
+          meta_partition.id(),
+          offset,
+          topic
+        );
+        tpl.add_partition_offset(topic, meta_partition.id(), offset)?;
+      }
+    }
+  } else {
+    anyhow::bail!("At least one of partition_offset or offset_model should be provided");
+  }
+  consumer.assign(&tpl)?;
+  Ok(())
+}

package/src/kafka/consumer/consumer_thread.rs

@@ -12,15 +12,14 @@ use tracing::{debug, error, info, warn};
 
 use crate::kafka::{
   consumer::kafka_consumer::RETRY_COUNTER_NAME,
-  kafka_util::{
-    kakfa_headers_to_hashmap, kakfa_headers_to_hashmap_buffer, ExtractValueOnKafkaHashMap,
-  },
-  model::Payload,
+  kafka_util::{create_payload, kakfa_headers_to_hashmap, ExtractValueOnKafkaHashMap},
+  producer::model::Payload,
 };
 
-use super::kafka_consumer::{ConsumerResult, ProducerHelper};
-
-use super::consumer_context::CustomContext;
+use super::{
+  kafka_consumer::{ConsumerResult, ProducerHelper},
+  model::CustomContext,
+};
 
 pub struct ConsumerThread {
   pub stream_consumer: StreamConsumer<CustomContext>,
@@ -180,17 +179,3 @@ impl ConsumerThread {
     }
   }
 }
-
-fn create_payload(message: &BorrowedMessage<'_>, payload: &[u8]) -> Payload {
-  let key: Option<Buffer> = message.key().map(|bytes| bytes.into());
-  let headers = Some(kakfa_headers_to_hashmap_buffer(message.headers()));
-  let payload_js = Payload::new(
-    payload.into(),
-    key,
-    headers,
-    message.topic().to_owned(),
-    message.partition(),
-    message.offset(),
-  );
-  payload_js
-}