kafka-crab-js 0.5.4 → 0.7.0

package/example/sample.mjs

@@ -34,7 +34,7 @@ async function produce() {
       const result = await producer.send(
         {
           topic,
-          messages: [{ value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`) }],
+          messages: [{ payload: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`) }],
         },
       );
       console.log('Js message sent. Offset:', result);

package/example/stream_consumer.mjs

@@ -0,0 +1,103 @@
+import { CommitMode, KafkaClient, KafkaStreamConsumer, PartitionPosition } from "../index.js";
+import { setTimeout as sleep } from 'node:timers/promises';
+import { fakerPT_BR  } from '@faker-js/faker';
+import { Readable } from "node:stream";
+
+const kafkaClient = new KafkaClient({
+    brokers: 'localhost:29092',
+    clientId: 'my-js-group-11',
+    logLevel: 'debug',
+    brokerAddressFamily: 'v4',
+  });
+
+/**
+ * Produces some messages to the topic
+ * @param {string} topic 
+ * @param {number} messages
+ */
+async function produce(topic, messages=1) {
+  const producer = kafkaClient.createProducer({ topic, configuration: { 'message.timeout.ms': '5000' } });
+  const index = Math.floor(Math.random() * 100_000)
+  for (let i = index ?? 0; i < index+messages; i++) {
+    try {
+      const result = await producer.send(
+        {
+          topic,
+          messages: [{ payload: Buffer.from(`{"_id":"${i}","name":"${fakerPT_BR.person.fullName()}","phone":"${fakerPT_BR.phone.number()}"}`) }],
+        },
+      );
+      console.log('Js message sent. Offset:', result);
+    } catch (error) {
+      console.error('Js Error on send', error);
+    }
+  }
+}
+
+class KafkaStreamReadable extends Readable {
+  /**
+   * Creates a new KafkaStream
+   * @param {KafkaStreamConsumer} kafkaStreamConsumer 
+   */
+  constructor(kafkaStreamConsumer) {
+    super({ objectMode: true });
+    this.kafkaStreamConsumer = kafkaStreamConsumer;
+  }
+
+  async _read() {
+    let message = await this.kafkaStreamConsumer.recv();
+    this.push(message)
+  }
+}
+
+async function startConsumer() {
+    const kafkaStreamConsumer = kafkaClient.createStreamConsumer({
+      groupId: `crab-test`,
+      enableAutoCommit: true,
+    });
+  
+    // await streamerConsumer.subscribe(
+    //   [{ 
+    //     topic: 'foo', 
+    //     partitionOffset: [
+    //       { partition:0, offset: { position: PartitionPosition.Beginning }} 
+    //     ]
+    //   }, 
+    //   { 
+    //     topic: 'bar', 
+    //     partitionOffset: [
+    //       { partition:0, offset: { position: PartitionPosition.Stored }},
+    //       { partition:1, offset: { position: PartitionPosition.Beginning }} 
+    //     ]
+    //   }]
+    // );
+    // await streamerConsumer.subscribe([ { topic: 'foo', allOffsets: { position: PartitionPosition.Stored } } ]);
+    // await streamerConsumer.subscribe('foo');
+    await kafkaStreamConsumer.subscribe([ { topic: 'foo' }, { topic: 'bar' } ]);
+    
+    const maxMessages = 20;
+    let counter = 0;
+    const kafkaStream = new KafkaStreamReadable(kafkaStreamConsumer);
+    console.log('Starting consumer')
+    kafkaStream.on('data', (message) => {
+      counter++;
+      console.log('>>> Message received:', { payload: message.payload.toString(), offset: message.offset, partition: message.partition, topic: message.topic })
+      if (counter === maxMessages) {
+        kafkaStream.destroy();
+      }
+      // streamerConsumer.commit(message.partition, message.offset+1, CommitMode.Sync)
+    })
+
+    kafkaStream.on('close', () => {
+      kafkaStreamConsumer.unsubscribe();
+      console.log('Stream ended')
+    })
+  }
+
+  if (process.argv[2] === 'send') {
+    const topic = process.argv[3]
+    const messages = process.argv[4] ? Number(process.argv[4]) : 1
+    console.log('Sending', messages, 'messages to', topic)
+    await produce(topic, messages);
+  } else { 
+    await startConsumer();
+  }

package/index.d.ts

@@ -3,10 +3,20 @@
 
 /* auto-generated by NAPI-RS */
 
-export const enum CommitMode {
-  AutoCommit = 'AutoCommit',
-  Sync = 'Sync',
-  Async = 'Async'
+export const enum ConsumerResult {
+  Ok = 'Ok',
+  Retry = 'Retry'
+}
+export interface KafkaConsumerConfiguration {
+  topic: string
+  groupId: string
+  retryStrategy?: RetryStrategy
+  offset?: OffsetModel
+  createTopic?: boolean
+  commitMode?: CommitMode
+  enableAutoCommit?: boolean
+  configuration?: Record<string, string>
+  fecthMetadataTimeout?: number
 }
 export interface RetryStrategy {
   retries: number
@@ -16,19 +26,34 @@ export interface RetryStrategy {
   offset?: OffsetModel
   configuration?: Record<string, string>
 }
-export const enum ConsumerResult {
-  Ok = 'Ok',
-  Retry = 'Retry'
+export const enum CommitMode {
+  Sync = 0,
+  Async = 1
 }
 export interface ConsumerConfiguration {
-  topic: string
   groupId: string
-  retryStrategy?: RetryStrategy
-  offset?: OffsetModel
   createTopic?: boolean
-  commitMode?: CommitMode
   enableAutoCommit?: boolean
   configuration?: Record<string, string>
+  fecthMetadataTimeout?: number
+}
+export const enum PartitionPosition {
+  Beginning = 'Beginning',
+  End = 'End',
+  Stored = 'Stored'
+}
+export interface OffsetModel {
+  offset?: number
+  position?: PartitionPosition
+}
+export interface PartitionOffset {
+  partition: number
+  offset: OffsetModel
+}
+export interface TopicPartitionConfig {
+  topic: string
+  allOffsets?: OffsetModel
+  partitionOffset?: Array<PartitionOffset>
 }
 export const enum SecurityProtocol {
   Plaintext = 'Plaintext',
@@ -49,19 +74,6 @@ export interface ProducerConfiguration {
   thrownOnError?: boolean
   configuration?: Record<string, string>
 }
-export interface ProducerRecord {
-  topic: string
-  messages: Array<MessageModel>
-}
-export const enum AutoOffsetReset {
-  Smallest = 'Smallest',
-  Earliest = 'Earliest',
-  Beginning = 'Beginning',
-  Largest = 'Largest',
-  Latest = 'Latest',
-  End = 'End',
-  Error = 'Error'
-}
 export interface Payload {
   value: Buffer
   key?: Buffer
@@ -70,18 +82,13 @@ export interface Payload {
   partition: number
   offset: number
 }
-export const enum PartitionPosition {
-  Beginning = 'Beginning',
-  End = 'End',
-  Stored = 'Stored'
-}
-export interface OffsetModel {
-  offset?: number
-  position?: PartitionPosition
-}
-export interface KafkaCrabError {
-  code: number
-  message: string
+export interface Message {
+  payload: Buffer
+  key?: Buffer
+  headers?: Record<string, Buffer>
+  topic: string
+  partition: number
+  offset: number
 }
 export interface RecordMetadata {
   topic: string
@@ -89,19 +96,35 @@ export interface RecordMetadata {
   offset: number
   error?: KafkaCrabError
 }
-export interface MessageModel {
-  value: Buffer
+export interface MessageProducer {
+  payload: Buffer
   key?: Buffer
   headers?: Record<string, Buffer>
 }
+export interface ProducerRecord {
+  topic: string
+  messages: Array<MessageProducer>
+}
+export interface KafkaCrabError {
+  code: number
+  message: string
+}
 export class KafkaConsumer {
   startConsumer(callback: (error: Error | undefined, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
 }
+export class KafkaStreamConsumer {
+  subscribe(topicConfigs: string | Array<TopicPartitionConfig>): Promise<void>
+  unsubscribe(): void
+  seek(topic: string, partition: number, offsetModel: OffsetModel, timeout?: number | undefined | null): void
+  recv(): Promise<Message>
+  commit(topic: string, partition: number, offset: number, commit: CommitMode): void
+}
 export class KafkaClient {
   readonly kafkaConfiguration: KafkaConfiguration
   constructor(kafkaConfiguration: KafkaConfiguration)
   createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
-  createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
+  createConsumer(consumerConfiguration: KafkaConsumerConfiguration): KafkaConsumer
+  createStreamConsumer(consumerConfiguration: ConsumerConfiguration): KafkaStreamConsumer
 }
 export class KafkaProducer {
   send(producerRecord: ProducerRecord): Promise<Array<RecordMetadata>>

package/index.js

@@ -224,17 +224,32 @@ switch (platform) {
         }
         break
       case 'arm':
-        localFileExisted = existsSync(
-          join(__dirname, 'kafka-crab-js.linux-arm-gnueabihf.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./kafka-crab-js.linux-arm-gnueabihf.node')
-          } else {
-            nativeBinding = require('kafka-crab-js-linux-arm-gnueabihf')
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-arm-musleabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-arm-musleabihf.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-arm-musleabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'kafka-crab-js.linux-arm-gnueabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./kafka-crab-js.linux-arm-gnueabihf.node')
+            } else {
+              nativeBinding = require('kafka-crab-js-linux-arm-gnueabihf')
+            }
+          } catch (e) {
+            loadError = e
           }
-        } catch (e) {
-          loadError = e
         }
         break
       case 'riscv64':
@@ -295,13 +310,13 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { CommitMode, ConsumerResult, KafkaConsumer, SecurityProtocol, KafkaClient, KafkaProducer, AutoOffsetReset, PartitionPosition } = nativeBinding
+const { ConsumerResult, KafkaConsumer, KafkaStreamConsumer, CommitMode, PartitionPosition, SecurityProtocol, KafkaClient, KafkaProducer } = nativeBinding
 
-module.exports.CommitMode = CommitMode
 module.exports.ConsumerResult = ConsumerResult
 module.exports.KafkaConsumer = KafkaConsumer
+module.exports.KafkaStreamConsumer = KafkaStreamConsumer
+module.exports.CommitMode = CommitMode
+module.exports.PartitionPosition = PartitionPosition
 module.exports.SecurityProtocol = SecurityProtocol
 module.exports.KafkaClient = KafkaClient
 module.exports.KafkaProducer = KafkaProducer
-module.exports.AutoOffsetReset = AutoOffsetReset
-module.exports.PartitionPosition = PartitionPosition

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.5.4",
+  "version": "0.7.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -29,8 +29,12 @@
     "url": "https://github.com/inaiat/kafka-crab-js.git"
   },
   "devDependencies": {
-    "@napi-rs/cli": "^2.18.0",
+    "@faker-js/faker": "^8.4.1",
+    "@napi-rs/cli": "^2.18.3",
+    "@types/node": "^20.12.12",
     "ava": "^6.1.2",
+    "nanoid": "^5.0.7",
+    "typescript": "^5.4.5",
     "xo": "^0.58.0"
   },
   "ava": {
@@ -55,10 +59,10 @@
     "universal": "napi universal",
     "version": "napi version"
   },
-  "packageManager": "yarn@4.1.1",
+  "packageManager": "pnpm@9.1.1",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.5.4",
-    "kafka-crab-js-darwin-x64": "0.5.4",
-    "kafka-crab-js-linux-x64-gnu": "0.5.4"
+    "kafka-crab-js-darwin-arm64": "0.7.0",
+    "kafka-crab-js-darwin-x64": "0.7.0",
+    "kafka-crab-js-linux-x64-gnu": "0.7.0"
   }
 }

package/src/kafka/consumer/consumer_helper.rs

@@ -0,0 +1,190 @@
+use std::{collections::HashMap, time::Duration};
+
+use rdkafka::{
+  config::RDKafkaLogLevel,
+  consumer::{Consumer, StreamConsumer},
+  ClientConfig, Offset, TopicPartitionList,
+};
+use tracing::{debug, error, info, warn};
+
+use crate::kafka::{consumer::model::LoggingConsumer, kafka_admin::KafkaAdmin};
+
+use super::model::{
+  ConsumerConfiguration, CustomContext, OffsetModel, PartitionOffset, PartitionPosition,
+};
+
+pub fn convert_to_rdkafka_offset(offset_model: &OffsetModel) -> Offset {
+  match offset_model.position {
+    Some(PartitionPosition::Beginning) => Offset::Beginning,
+    Some(PartitionPosition::End) => Offset::End,
+    Some(PartitionPosition::Stored) => Offset::Stored,
+    None => match offset_model.offset {
+      Some(value) => Offset::Offset(value),
+      None => Offset::Stored, // Default to stored
+    },
+  }
+}
+
+pub async fn create_stream_consumer_and_setup_everything(
+  client_config: &ClientConfig,
+  consumer_configuration: &ConsumerConfiguration,
+  topic: &str,
+  offset: &Option<OffsetModel>,
+  configuration: Option<HashMap<String, String>>,
+  timeout: Duration,
+) -> anyhow::Result<StreamConsumer<CustomContext>> {
+  let consumer = create_stream_consumer(client_config, consumer_configuration, configuration)?;
+
+  if consumer_configuration.create_topic.unwrap_or(true) {
+    try_create_topic(&vec![topic.to_owned()], client_config, timeout).await?;
+  }
+
+  if let Some(offset_model) = offset {
+    set_offset_of_all_partitions(offset_model, &consumer, topic, timeout)?;
+  }
+
+  try_subscribe(&consumer, &vec![topic.to_owned()])?;
+
+  Ok(consumer)
+}
+
+pub fn create_stream_consumer(
+  client_config: &ClientConfig,
+  consumer_configuration: &ConsumerConfiguration,
+  configuration: Option<HashMap<String, String>>,
+) -> anyhow::Result<StreamConsumer<CustomContext>> {
+  let context = CustomContext;
+
+  let ConsumerConfiguration {
+    group_id,
+    enable_auto_commit,
+    ..
+  } = consumer_configuration.clone();
+
+  let mut consumer_config: ClientConfig = client_config.clone();
+
+  if let Some(config) = configuration {
+    consumer_config.extend(config);
+  }
+
+  debug!(
+    "Creating consumer with configuration: {:?}",
+    consumer_config
+  );
+
+  let consumer: LoggingConsumer = consumer_config
+    .clone()
+    .set("group.id", group_id.clone())
+    .set(
+      "enable.auto.commit",
+      enable_auto_commit.unwrap_or(true).to_string(),
+    )
+    .set_log_level(RDKafkaLogLevel::Debug)
+    .create_with_context(context)?;
+
+  info!("Consumer created. Group id: {:?}", group_id);
+  Ok(consumer)
+}
+
+pub fn try_subscribe(consumer: &LoggingConsumer, topics: &Vec<String>) -> anyhow::Result<()> {
+  let topics_ref = topics.iter().map(|x| x.as_str()).collect::<Vec<&str>>();
+  consumer.subscribe(topics_ref.as_slice()).map_err(|e| {
+    anyhow::Error::msg(format!(
+      "Can't subscribe to specified topic(s): {:?}. Error: {:?}",
+      topics_ref, e
+    ))
+  })?;
+  debug!("Subscribed to topic(s): {:?}", topics_ref);
+  Ok(())
+}
+
+pub async fn try_create_topic(
+  topics: &Vec<String>,
+  client_config: &ClientConfig,
+  fetch_metadata_timeout: Duration,
+) -> anyhow::Result<()> {
+  let admin = KafkaAdmin::new(client_config, Some(fetch_metadata_timeout));
+  let result = admin.create_topic(topics).await;
+  if let Err(e) = result {
+    warn!("Fail to create topic {:?}", e);
+    return Err(anyhow::Error::msg(format!("Fail to create topic: {:?}", e)));
+  }
+  info!("Topic(s) created: {:?}", topics);
+  Ok(())
+}
+
+pub fn set_offset_of_all_partitions(
+  offset_model: &OffsetModel,
+  consumer: &StreamConsumer<CustomContext>,
+  topic: &str,
+  timeout: Duration,
+) -> anyhow::Result<()> {
+  let offset = convert_to_rdkafka_offset(offset_model);
+  debug!("Setting offset to: {:?}", offset);
+  let metadata = consumer.fetch_metadata(Some(topic), timeout)?;
+
+  metadata.topics().iter().for_each(|meta_topic| {
+    let mut tpl = TopicPartitionList::new();
+    meta_topic.partitions().iter().for_each(|meta_partition| {
+      info!("Adding partition: {:?}", meta_partition.id());
+      tpl.add_partition(topic, meta_partition.id());
+    });
+    match tpl.set_all_offsets(offset) {
+      Ok(_) => {
+        debug!("Offset set to: {:?}", offset);
+      }
+      Err(e) => {
+        error!("Fail to set offset: {:?}", e)
+      }
+    };
+    match consumer.assign(&tpl) {
+      Ok(_) => {
+        debug!("Assigning topic: {:?}", topic);
+      }
+      Err(e) => {
+        error!("Fail to assign topic: {:?}", e);
+      }
+    }
+  });
+
+  Ok(())
+}
+
+pub fn assign_offset_or_use_metadata(
+  topic: &str,
+  partition_offset: Option<Vec<PartitionOffset>>,
+  offset_model: Option<&OffsetModel>,
+  consumer: &StreamConsumer<CustomContext>,
+  timeout: Duration,
+) -> anyhow::Result<()> {
+  let mut tpl = TopicPartitionList::new();
+
+  if let Some(value) = partition_offset {
+    for item in value {
+      let offset = convert_to_rdkafka_offset(&item.offset);
+      debug!(
+        "Adding partition: {:?} with offset: {:?} for topic: {:?}",
+        item.partition, offset, topic
+      );
+      tpl.add_partition_offset(topic, item.partition, offset)?;
+    }
+  } else if let Some(offset_model) = offset_model {
+    let offset = convert_to_rdkafka_offset(offset_model);
+    let metadata = consumer.fetch_metadata(Some(topic), timeout)?;
+    for meta_topic in metadata.topics() {
+      for meta_partition in meta_topic.partitions() {
+        info!(
+          "Adding partition: {:?} with offset: {:?} for topic: {:?}",
+          meta_partition.id(),
+          offset,
+          topic
+        );
+        tpl.add_partition_offset(topic, meta_partition.id(), offset)?;
+      }
+    }
+  } else {
+    anyhow::bail!("At least one of partition_offset or offset_model should be provided");
+  }
+  consumer.assign(&tpl)?;
+  Ok(())
+}

package/src/kafka/consumer/consumer_thread.rs

@@ -15,12 +15,13 @@ use crate::kafka::{
   kafka_util::{
     kakfa_headers_to_hashmap, kakfa_headers_to_hashmap_buffer, ExtractValueOnKafkaHashMap,
   },
-  model::Payload,
+  producer::model::Payload,
 };
 
-use super::kafka_consumer::{ConsumerResult, ProducerHelper};
-
-use super::consumer_context::CustomContext;
+use super::{
+  kafka_consumer::{ConsumerResult, ProducerHelper},
+  model::CustomContext,
+};
 
 pub struct ConsumerThread {
   pub stream_consumer: StreamConsumer<CustomContext>,
@@ -181,7 +182,7 @@ impl ConsumerThread {
   }
 }
 
-fn create_payload(message: &BorrowedMessage<'_>, payload: &[u8]) -> Payload {
+pub fn create_payload(message: &BorrowedMessage<'_>, payload: &[u8]) -> Payload {
   let key: Option<Buffer> = message.key().map(|bytes| bytes.into());
   let headers = Some(kakfa_headers_to_hashmap_buffer(message.headers()));
   let payload_js = Payload::new(