kafka-crab-js 0.0.9 → 0.1.0

package/Cargo.toml

@@ -8,15 +8,16 @@ crate-type = ["cdylib"]
 
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2.13", default-features = false, features = [
-    "napi4",
+napi = { version = "2", default-features = false, features = [
+    "napi8",
     "tokio_rt",
+    "serde-json",
 ] }
-napi-derive = "2.13"
+napi-derive = "2"
 
 anyhow = { version = "1.0", features = ["backtrace"] }
 
-rdkafka = { version = "0.32", features = [
+rdkafka = { version = "0.33", features = [
     "libz-static",
     "cmake-build",
     "tokio",

package/README.md

@@ -1,16 +1,77 @@
-Kafka Crab Js
--------------
+# Kafka Crab Js
 
 **Blazingly Fast Kafka Client for NodeJs**
 
 Kafka-Crab-JS is a powerful Node.js library that allows developers to interact with Apache Kafka using Rust programming language, seamlessly integrated with Node.js applications. This documentation provides an overview of the Kafka-Crab-JS library, including installation instructions, usage guidelines, and examples.
 
+The plugin can be found on [npm](https://www.npmjs.com/package/kafka-crab-js) and the source code is available on [GitHub](https://github.com/inaiat/kafka-crab-js).
 
-# Installation
+## Table of Contents
+1. [Installation](#installation)
+2. [Usage](#usage)
+   - [Client](#client)
+   - [Producer](#producer)
+   - [Consumer](#consumer)
+3. [Examples](#examples)
 
-To use Kafka-Crab-JS in your Node.js project, follow the steps below:
 
-todo
+## Installation
+To use Kafka Crab JS, you need to have Node.js and npm (Node Package Manager) installed on your system. Once you have them set up, you can install the plugin by running the following command:
 
-# Examples
-todo
+```shell
+npm install kafka-crab-js
+```
+## Usage
+### Client
+Creating a Client Instance
+Next, you can create a client instance by specifying the Kafka broker(s) and other optional configurations:
+
+```javascript
+import { KafkaClient }
+
+const kafkaClient = new KafkaClient({
+  brokers: 'localhost:29092',
+  clientId: 'my-id'});
+```
+### Producer
+The producer component allows you to send messages to Kafka topics.
+
+```javascript
+const result = await producer.send({
+    topic: 'my-js-topic',
+    key: Buffer.from('abc'),
+    value: Buffer.from(`{"name":"Elizeu Drummond","phone":"55219123456"}`),
+    headers: {key: Buffer.from('value1')},
+});
+console.log('Message sent with offset:', result);
+```
+
+
+### Consumer
+
+**Creating a Consumer Instance**
+
+Create a consumer instance by specifying the Kafka broker(s), group ID, and other optional configurations:
+
+```javascript
+const consumer = kafkaClient.createConsumer({
+  topic: 'my-js-topic',
+  groupId: 'my-js-group',
+  retryStrategy: {
+    retries: 3,
+  },
+});
+```
+
+You can consume messages using the run startConsumer:
+
+```javascript
+consumer.startConsumer(async (error, value) => {
+  const message = JSON.parse(value.toString());
+  const content = await process_my_message(message);
+  return ConsumerResult.Ok;
+});
+```
+## Examples
+
+You can find examples of using Kafka Crab JS in the [GitHub repository](https://github.com/inaiat/kafka-crab-js).

package/example/sample.mjs

@@ -1,73 +1,67 @@
 /* eslint-disable no-await-in-loop */
+
 import timersPromises from 'node:timers/promises';
 import {Buffer} from 'node:buffer';
-import {ConsumerResult, KafkaClient, KafkaCommitMode, PartitionPosition} from '../index.js';
-
-const kafkaClient = new KafkaClient({ 
-  brokers: 'localhost:29092', 
-  clientId: 'my-id', 
-  enableAnsiLogger: false});
-const topic = 'my-js-topic';
-
-let counter = 0;
+import {nanoid} from 'nanoid';
+import {KafkaClient, CommitMode, PartitionPosition, ConsumerResult} from '../index.js';
 
-async function process(message) {
-  await timersPromises.setTimeout(1);
-  return 'message received: ' + JSON.stringify(message);
-}
+const kafkaClient = new KafkaClient({
+  brokers: 'localhost:29092',
+  clientId: 'my-ids',
+  groupId: 'my-js-group',
+  enableAnsiLogger: true});
+const topic = `topic-${nanoid()}`;
 
 const consumer = kafkaClient.createConsumer({
   topic,
   groupId: 'my-js-group',
-  commitMode: KafkaCommitMode.Sync,
+  CommitMode: CommitMode.AutoCommit,
   offset: {position: PartitionPosition.Stored},
+  configuration: {'auto.offset.reset': 'earliest'},
   retryStrategy: {
-    retries: 3,
+    retries: 5,
+    pauseConsumerDuration: 5000,
   },
 });
+const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '5000'}});
 
-console.time('consumer');
-
-consumer.startConsumer(async (error, value) => {
-  if (error) {
-    console.error('Js Consumer error', error);
-    return;
-  }
-
-  const message = JSON.parse(value.toString());
-  const content = await process(message);
-  counter++;
-  if (counter >= 0) {
-    console.timeEnd('consumer');
-    console.log('Js Counter:', counter, 'Content:', JSON.stringify(content));
-    console.time('consumer');
+async function produce() {
+  for (let i = 0; i < 10; i++) {
+    try {
+      const result = await producer.send(
+        {
+          topic,
+          messages: [{value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`)}],
+        },
+      );
+      console.log('Js message sent. Offset:', result);
+    } catch (error) {
+      console.error('Js Error on send', error);
+    }
   }
+}
 
-  if (message._id === '3') {
-    console.error('Js Error on process message, let\'s retry', JSON.stringify(content));
-    return ConsumerResult.Retry;
-  }
+async function startConsumer() {
+  consumer.startConsumer(async (error, {value, partition, offset}) => {
+    if (error) {
+      console.error('Js Consumer error', error);
+      return;
+    }
 
+    const message = JSON.parse(value.toString());
 
-  return ConsumerResult.Ok;
-});
+    console.log('Message received! Partition:', partition, 'Offset:', offset, 'Message =>', message);
 
-const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '5000'}});
-console.log('Sending message');
+    if (message._id === '5') {
+      return ConsumerResult.Retry;
+    }
 
-for (let i = 0; i < 5; i++) {
-  try {
-    const result = await producer.send(
-      {
-        topic,
-        key: Buffer.from('abc'),
-        value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`),
-        headers: {key: Buffer.from('value1')},
-      },
-    );
-    console.log("Js message sent. Offset: ", result);
-    await timersPromises.setTimeout(1200);
-  } catch (error) {
-    console.error('Js Error on send', error);
-  }
+    return ConsumerResult.Ok;
+  });
 }
+
+console.time('consumer');
+
+await startConsumer();
+
+await produce();

package/index.d.ts

@@ -16,27 +16,37 @@ export interface KafkaConfiguration {
   configuration?: Record<string, string>
   enableAnsiLogger?: boolean
 }
+export const enum CommitMode {
+  AutoCommit = 'AutoCommit',
+  Sync = 'Sync',
+  Async = 'Async'
+}
+export interface RetryStrategy {
+  retries: number
+  retryTopic?: string
+  dqlTopic?: string
+  pauseConsumerDuration?: number
+}
+export const enum ConsumerResult {
+  Ok = 'Ok',
+  Retry = 'Retry'
+}
 export interface ConsumerConfiguration {
   topic: string
   groupId: string
   retryStrategy?: RetryStrategy
   offset?: OffsetModel
   createTopic?: boolean
-  commitMode?: KafkaCommitMode
+  commitMode?: CommitMode
   enableAutoCommit?: boolean
   configuration?: Record<string, string>
 }
 export interface ProducerConfiguration {
   queueTimeout?: number
+  thrownOnError?: boolean
   configuration?: Record<string, string>
 }
-export interface MessageModel {
-  topic: string
-  key: Buffer
-  value: Buffer
-  headers?: Record<string, Buffer>
-}
-export interface ProduceRecord {
+export interface ProducerRecord {
   topic: string
   messages: Array<MessageModel>
 }
@@ -49,12 +59,11 @@ export const enum AutoOffsetReset {
   End = 'End',
   Error = 'Error'
 }
-export const enum KafkaCommitMode {
-  AutoCommit = 'AutoCommit',
-  Sync = 'Sync',
-  Async = 'Async'
-}
-export interface OwnedDelivery {
+export interface Payload {
+  value: Buffer
+  key?: Buffer
+  headers?: Record<string, Buffer>
+  topic: string
   partition: number
   offset: number
 }
@@ -67,15 +76,20 @@ export interface OffsetModel {
   offset?: number
   position?: PartitionPosition
 }
-export interface RetryStrategy {
-  retries: number
-  retryTopic?: string
-  dqlTopic?: string
-  pauseConsumerDuration?: number
+export interface KafkaCrabError {
+  code: number
+  message: string
 }
-export const enum ConsumerResult {
-  Ok = 'Ok',
-  Retry = 'Retry'
+export interface RecordMetadata {
+  topic: string
+  partition: number
+  offset: number
+  error?: KafkaCrabError
+}
+export interface MessageModel {
+  value: Buffer
+  key?: Buffer
+  headers?: Record<string, Buffer>
 }
 export class KafkaClient {
   readonly kafkaConfiguration: KafkaConfiguration
@@ -83,9 +97,10 @@ export class KafkaClient {
   createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
   createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
 }
+export class ProducerHelper { }
 export class KafkaConsumer {
-  startConsumer(callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult | undefined>): Promise<void>
+  startConsumer(callback: (err: Error | null, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
 }
 export class KafkaProducer {
-  send(message: MessageModel): Promise<OwnedDelivery>
+  send(producerRecord: ProducerRecord): Promise<Array<RecordMetadata>>
 }

package/index.js

@@ -252,13 +252,14 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SecurityProtocol, KafkaClient, KafkaConsumer, KafkaProducer, AutoOffsetReset, KafkaCommitMode, PartitionPosition, ConsumerResult } = nativeBinding
+const { SecurityProtocol, KafkaClient, CommitMode, ConsumerResult, ProducerHelper, KafkaConsumer, KafkaProducer, AutoOffsetReset, PartitionPosition } = nativeBinding
 
 module.exports.SecurityProtocol = SecurityProtocol
 module.exports.KafkaClient = KafkaClient
+module.exports.CommitMode = CommitMode
+module.exports.ConsumerResult = ConsumerResult
+module.exports.ProducerHelper = ProducerHelper
 module.exports.KafkaConsumer = KafkaConsumer
 module.exports.KafkaProducer = KafkaProducer
 module.exports.AutoOffsetReset = AutoOffsetReset
-module.exports.KafkaCommitMode = KafkaCommitMode
 module.exports.PartitionPosition = PartitionPosition
-module.exports.ConsumerResult = ConsumerResult

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.0.9",
+  "version": "0.1.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -27,6 +27,8 @@
   "devDependencies": {
     "@napi-rs/cli": "^2.16.1",
     "ava": "^5.3.1",
+    "benny": "^3.7.1",
+    "kafkajs": "^2.2.4",
     "nanoid": "^4.0.2",
     "xo": "^0.54.2"
   },
@@ -37,7 +39,10 @@
     "node": ">= 10"
   },
   "xo": {
-    "space": true
+    "space": true,
+    "rules": {
+      "new-cap": "off"
+    }
   },
   "scripts": {
     "artifacts": "napi artifacts",
@@ -50,8 +55,8 @@
   },
   "packageManager": "yarn@3.6.0",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.0.9",
-    "kafka-crab-js-darwin-x64": "0.0.9",
-    "kafka-crab-js-linux-x64-gnu": "0.0.9"
+    "kafka-crab-js-darwin-arm64": "0.1.0",
+    "kafka-crab-js-darwin-x64": "0.1.0",
+    "kafka-crab-js-linux-x64-gnu": "0.1.0"
   }
 }