kafka-crab-js 0.0.10 → 0.1.0

package/Cargo.toml

@@ -8,15 +8,16 @@ crate-type = ["cdylib"]
 
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2.13", default-features = false, features = [
-    "napi4",
+napi = { version = "2", default-features = false, features = [
+    "napi8",
     "tokio_rt",
+    "serde-json",
 ] }
-napi-derive = "2.13"
+napi-derive = "2"
 
 anyhow = { version = "1.0", features = ["backtrace"] }
 
-rdkafka = { version = "0.32", features = [
+rdkafka = { version = "0.33", features = [
     "libz-static",
     "cmake-build",
     "tokio",

package/example/sample.mjs

@@ -1,73 +1,67 @@
 /* eslint-disable no-await-in-loop */
+
 import timersPromises from 'node:timers/promises';
 import {Buffer} from 'node:buffer';
-import {ConsumerResult, KafkaClient, KafkaCommitMode, PartitionPosition} from '../index.js';
-
-const kafkaClient = new KafkaClient({ 
-  brokers: 'localhost:29092', 
-  clientId: 'my-id', 
-  enableAnsiLogger: false});
-const topic = 'my-js-topic';
-
-let counter = 0;
+import {nanoid} from 'nanoid';
+import {KafkaClient, CommitMode, PartitionPosition, ConsumerResult} from '../index.js';
 
-async function process(message) {
-  await timersPromises.setTimeout(1);
-  return 'message received: ' + JSON.stringify(message);
-}
+const kafkaClient = new KafkaClient({
+  brokers: 'localhost:29092',
+  clientId: 'my-ids',
+  groupId: 'my-js-group',
+  enableAnsiLogger: true});
+const topic = `topic-${nanoid()}`;
 
 const consumer = kafkaClient.createConsumer({
   topic,
   groupId: 'my-js-group',
-  commitMode: KafkaCommitMode.Sync,
+  CommitMode: CommitMode.AutoCommit,
   offset: {position: PartitionPosition.Stored},
+  configuration: {'auto.offset.reset': 'earliest'},
   retryStrategy: {
-    retries: 3,
+    retries: 5,
+    pauseConsumerDuration: 5000,
   },
 });
+const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '5000'}});
 
-console.time('consumer');
-
-consumer.startConsumer(async (error, value) => {
-  if (error) {
-    console.error('Js Consumer error', error);
-    return;
-  }
-
-  const message = JSON.parse(value.toString());
-  const content = await process(message);
-  counter++;
-  if (counter >= 0) {
-    console.timeEnd('consumer');
-    console.log('Js Counter:', counter, 'Content:', JSON.stringify(content));
-    console.time('consumer');
+async function produce() {
+  for (let i = 0; i < 10; i++) {
+    try {
+      const result = await producer.send(
+        {
+          topic,
+          messages: [{value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`)}],
+        },
+      );
+      console.log('Js message sent. Offset:', result);
+    } catch (error) {
+      console.error('Js Error on send', error);
+    }
   }
+}
 
-  if (message._id === '3') {
-    console.error('Js Error on process message, let\'s retry', JSON.stringify(content));
-    return ConsumerResult.Retry;
-  }
+async function startConsumer() {
+  consumer.startConsumer(async (error, {value, partition, offset}) => {
+    if (error) {
+      console.error('Js Consumer error', error);
+      return;
+    }
 
+    const message = JSON.parse(value.toString());
 
-  return ConsumerResult.Ok;
-});
+    console.log('Message received! Partition:', partition, 'Offset:', offset, 'Message =>', message);
 
-const producer = kafkaClient.createProducer({topic, configuration: {'message.timeout.ms': '5000'}});
-console.log('Sending message');
+    if (message._id === '5') {
+      return ConsumerResult.Retry;
+    }
 
-for (let i = 0; i < 5; i++) {
-  try {
-    const result = await producer.send(
-      {
-        topic,
-        key: Buffer.from('abc'),
-        value: Buffer.from(`{"_id":"${i}","name":"Elizeu Drummond Sample js","phone":"555"}`),
-        headers: {key: Buffer.from('value1')},
-      },
-    );
-    console.log("Js message sent. Offset: ", result);
-    await timersPromises.setTimeout(1200);
-  } catch (error) {
-    console.error('Js Error on send', error);
-  }
+    return ConsumerResult.Ok;
+  });
 }
+
+console.time('consumer');
+
+await startConsumer();
+
+await produce();

package/index.d.ts

@@ -16,27 +16,37 @@ export interface KafkaConfiguration {
   configuration?: Record<string, string>
   enableAnsiLogger?: boolean
 }
+export const enum CommitMode {
+  AutoCommit = 'AutoCommit',
+  Sync = 'Sync',
+  Async = 'Async'
+}
+export interface RetryStrategy {
+  retries: number
+  retryTopic?: string
+  dqlTopic?: string
+  pauseConsumerDuration?: number
+}
+export const enum ConsumerResult {
+  Ok = 'Ok',
+  Retry = 'Retry'
+}
 export interface ConsumerConfiguration {
   topic: string
   groupId: string
   retryStrategy?: RetryStrategy
   offset?: OffsetModel
   createTopic?: boolean
-  commitMode?: KafkaCommitMode
+  commitMode?: CommitMode
   enableAutoCommit?: boolean
   configuration?: Record<string, string>
 }
 export interface ProducerConfiguration {
   queueTimeout?: number
+  thrownOnError?: boolean
   configuration?: Record<string, string>
 }
-export interface MessageModel {
-  topic: string
-  key: Buffer
-  value: Buffer
-  headers?: Record<string, Buffer>
-}
-export interface ProduceRecord {
+export interface ProducerRecord {
   topic: string
   messages: Array<MessageModel>
 }
@@ -49,12 +59,11 @@ export const enum AutoOffsetReset {
   End = 'End',
   Error = 'Error'
 }
-export const enum KafkaCommitMode {
-  AutoCommit = 'AutoCommit',
-  Sync = 'Sync',
-  Async = 'Async'
-}
-export interface OwnedDelivery {
+export interface Payload {
+  value: Buffer
+  key?: Buffer
+  headers?: Record<string, Buffer>
+  topic: string
   partition: number
   offset: number
 }
@@ -67,15 +76,20 @@ export interface OffsetModel {
   offset?: number
   position?: PartitionPosition
 }
-export interface RetryStrategy {
-  retries: number
-  retryTopic?: string
-  dqlTopic?: string
-  pauseConsumerDuration?: number
+export interface KafkaCrabError {
+  code: number
+  message: string
 }
-export const enum ConsumerResult {
-  Ok = 'Ok',
-  Retry = 'Retry'
+export interface RecordMetadata {
+  topic: string
+  partition: number
+  offset: number
+  error?: KafkaCrabError
+}
+export interface MessageModel {
+  value: Buffer
+  key?: Buffer
+  headers?: Record<string, Buffer>
 }
 export class KafkaClient {
   readonly kafkaConfiguration: KafkaConfiguration
@@ -83,9 +97,10 @@ export class KafkaClient {
   createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
   createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer
 }
+export class ProducerHelper { }
 export class KafkaConsumer {
-  startConsumer(callback: (err: Error | null, result: Buffer) => Promise<ConsumerResult | undefined>): Promise<void>
+  startConsumer(callback: (err: Error | null, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
 }
 export class KafkaProducer {
-  send(message: MessageModel): Promise<OwnedDelivery>
+  send(producerRecord: ProducerRecord): Promise<Array<RecordMetadata>>
 }

package/index.js

@@ -252,13 +252,14 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SecurityProtocol, KafkaClient, KafkaConsumer, KafkaProducer, AutoOffsetReset, KafkaCommitMode, PartitionPosition, ConsumerResult } = nativeBinding
+const { SecurityProtocol, KafkaClient, CommitMode, ConsumerResult, ProducerHelper, KafkaConsumer, KafkaProducer, AutoOffsetReset, PartitionPosition } = nativeBinding
 
 module.exports.SecurityProtocol = SecurityProtocol
 module.exports.KafkaClient = KafkaClient
+module.exports.CommitMode = CommitMode
+module.exports.ConsumerResult = ConsumerResult
+module.exports.ProducerHelper = ProducerHelper
 module.exports.KafkaConsumer = KafkaConsumer
 module.exports.KafkaProducer = KafkaProducer
 module.exports.AutoOffsetReset = AutoOffsetReset
-module.exports.KafkaCommitMode = KafkaCommitMode
 module.exports.PartitionPosition = PartitionPosition
-module.exports.ConsumerResult = ConsumerResult

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-crab-js",
-  "version": "0.0.10",
+  "version": "0.1.0",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -27,6 +27,8 @@
   "devDependencies": {
     "@napi-rs/cli": "^2.16.1",
     "ava": "^5.3.1",
+    "benny": "^3.7.1",
+    "kafkajs": "^2.2.4",
     "nanoid": "^4.0.2",
     "xo": "^0.54.2"
   },
@@ -37,7 +39,10 @@
     "node": ">= 10"
   },
   "xo": {
-    "space": true
+    "space": true,
+    "rules": {
+      "new-cap": "off"
+    }
   },
   "scripts": {
     "artifacts": "napi artifacts",
@@ -50,8 +55,8 @@
   },
   "packageManager": "yarn@3.6.0",
   "optionalDependencies": {
-    "kafka-crab-js-darwin-arm64": "0.0.10",
-    "kafka-crab-js-darwin-x64": "0.0.10",
-    "kafka-crab-js-linux-x64-gnu": "0.0.10"
+    "kafka-crab-js-darwin-arm64": "0.1.0",
+    "kafka-crab-js-darwin-x64": "0.1.0",
+    "kafka-crab-js-linux-x64-gnu": "0.1.0"
   }
 }