kafka-crab-js 0.7.0 → 1.0.0

package/README.md

@@ -1,75 +1,254 @@
-# Kafka Crab Js
+<div align="center">
 
-Kafka-Crab-JS is a powerful Node.js library that allows developers to interact with Apache Kafka using Rust programming language, seamlessly integrated with Node.js applications. This documentation provides an overview of the Kafka-Crab-JS library, including installation instructions, usage guidelines, and examples.
+# 🦀 Kafka Crab JS 🦀
 
-The plugin can be found on [npm](https://www.npmjs.com/package/kafka-crab-js) and the source code is available on [GitHub](https://github.com/inaiat/kafka-crab-js).
+A lightweight and flexible Kafka client for JavaScript/TypeScript, built with Rust-inspired reliability.
+
+[![npm version](https://img.shields.io/npm/v/kafka-crab-js.svg)](https://www.npmjs.com/package/kafka-crab-js)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+</div>
+
+## Features
+
+- 🦀 Simple and intuitive API
+- 🚀 High-performance message processing
+- 🔄 Automatic reconnection handling
+- 🎯 Type-safe interfaces (TypeScript support)
+- ⚡ Async/await support
+- 🛠️ Configurable consumer and producer options
+- 📊 Stream processing support
+- 📦 Message batching capabilities
+- 🔍 Comprehensive error handling
 
 ## Table of Contents
-1. [Installation](#installation)
-2. [Usage](#usage)
-   - [Client](#client)
-   - [Producer](#producer)
-   - [Consumer](#consumer)
-3. [Examples](#examples)
 
+1. [Installation](#installation)
+2. [Quick Start](#quick-start)
+3. [Consumer Examples](#consumer-examples)
+4. [Producer Examples](#producer-examples)
+5. [Stream Processing](#stream-processing)
+6. [Configuration](#configuration)
+7. [Best Practices](#best-practices)
+8. [Contributing](#contributing)
+9. [License](#license)
 
 ## Installation
-To use Kafka Crab JS, you need to have Node.js and npm (Node Package Manager) installed on your system. Once you have them set up, you can install the plugin by running the following command:
 
-```shell
+```bash
 npm install kafka-crab-js
+# or
+yarn add kafka-crab-js
 ```
-## Usage
-### Client
-Creating a Client Instance
-Next, you can create a client instance by specifying the Kafka broker(s) and other optional configurations:
+
+## Quick Start
+
+### Basic Consumer Setup
 
 ```javascript
-import { KafkaClient }
+import { KafkaClient } from 'kafka-crab-js';
+async function run() {
+  const kafkaClient = new KafkaClient({
+    brokers: 'localhost:29092',
+    clientId: 'foo-client',
+    logLevel: 'debug',
+    brokerAddressFamily: 'v4',
+  });
+
+  // Create consumer
+  const consumer = kafkaClient.createConsumer({
+    groupId: 'foo-group',
+  });
+
+  await consumer.subscribe([{ topic: 'foo' }]);
+
+  const message = await consumer.recv();
+  const { payload, partition, offset } = message;
+  console.log({
+    partition,
+    offset,
+    value: payload.toString()
+  });
+
+  consumer.unsubscribe();
+
+}
+
+await run();
+```
+
+### Basic Producer Setup
+
+```javascript
+import { KafkaClient } from 'kafka-crab-js';
 
 const kafkaClient = new KafkaClient({
   brokers: 'localhost:29092',
-  clientId: 'my-id'});
-```
-### Producer
-The producer component allows you to send messages to Kafka topics.
+  clientId: 'my-client-id',
+  logLevel: 'info',
+  brokerAddressFamily: 'v4',
+});
 
-```javascript
-const producer = kafkaClient.createProducer({topic: 'my-topic'});
+const producer = kafkaClient.createProducer({ configuration: { 'message.timeout.ms': '5000' } });
+
+const message = {
+  id: 1,
+  name: "Sample Message",
+  timestamp: new Date().toISOString()
+};
 
 const result = await producer.send({
-    topic: 'my-js-topic',
-    messages: [{value: Buffer.from(`{"name":"Elizeu Drummond","phone":"55219123456"}`)}],
+  topic: 'my-topic',
+  messages: [{
+    payload: Buffer.from(JSON.stringify(message))
+  }]
 });
-console.log('Message sent with offset:', result);
-```
 
+const errors = result.map(r => r.error).filter(Boolean);
+if (errors.length > 0) {
+  console.error('Error sending message:', errors);
+} else {
+  console.log('Message sent. Offset:', result);
+}
 
-### Consumer
+```
 
-**Creating a Consumer Instance**
+## Stream Processing
 
-Create a consumer instance by specifying the Kafka broker(s), group ID, and other optional configurations:
+### Stream Consumer Example
 
 ```javascript
-const consumer = kafkaClient.createConsumer({
-  topic: 'my-topic',
-  groupId: 'my-group',
-  retryStrategy: {
-    retries: 3,
-  },
+import { KafkaClient } from 'kafka-crab-js';
+
+const kafkaClient = new KafkaClient({
+  brokers: 'localhost:29092',
+  clientId: 'my-client-id',
+  logLevel: 'info',
+  brokerAddressFamily: 'v4',
+});
+
+const kafkaStream = kafkaClient.createStreamConsumer({
+  groupId: `my-groud-id`,
+  enableAutoCommit: true,
 });
+
+await kafkaStream.subscribe([{ topic: 'foo' }, { topic: 'bar' }])
+
+kafkaStream.on('data', (message) => {
+  console.log('>>> Message received:', { payload: message.payload.toString(), offset: message.offset, partition: message.partition, topic: message.topic })
+  if (message.offset > 10) {
+    kafkaStream.destroy();
+  }
+})
+
+kafkaStream.on('close', () => {
+  kafkaStream.unsubscribe();
+  console.log('Stream ended')
+})
 ```
 
-You can consume messages using the run startConsumer:
+## Producer Examples
 
-```javascript
-consumer.startConsumer(async (error, {value, partition, offset}) => {
-  const message = JSON.parse(value.toString());
-  console.log('Message received! Partition:', partition, 'Offset:', offset, 'Message =>', message.toString());
+### Batch Message Production
 
-  return ConsumerResult.Ok;
+```javascript
+const kafkaClient = new KafkaClient({
+  brokers: 'localhost:29092',
+  clientId: 'my-client-id',
+  brokerAddressFamily: 'v4',
 });
+const producer = kafkaClient.createProducer({});
+
+const messages = Array.from({ length: 100 }, (_, i) => ({
+  payload: Buffer.from(JSON.stringify({
+    _id: i,
+    name: `Batch Message ${i}`,
+    timestamp: new Date().toISOString()
+  }))
+}));
+
+try {
+  const result = await producer.send({
+    topic: 'my-topic',
+    messages
+  });
+  console.log('Batch sent. Offset:', result);
+  console.assert(result.length === 100);
+} catch (error) {
+  console.error('Batch error:', error);
+}
 ```
-## Examples
-You can find examples of using Kafka Crab JS in the [GitHub repository](https://github.com/inaiat/kafka-crab-js).
+
+### Producer with Keys and Headers
+
+```javascript
+async function produceWithMetadata() {
+  const producer = await kafkaCrab.createProducer({ config });
+
+  try {
+    await producer.send({
+      topic,
+      messages: [{
+        key: 'user-123',
+        payload: Buffer.from(JSON.stringify({
+          userId: 123,
+          action: 'update'
+        })),
+        headers: {
+          'correlation-id': 'txn-123',
+          'source': 'user-service'
+        }
+      }]
+    });
+  } catch (error) {
+    console.error('Error:', error);
+  }
+}
+```
+
+## Configuration
+
+### Configuration properties
+
+You can see the available options here: [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
+
+## Best Practices
+
+### Error Handling
+- Always wrap async operations in try-catch blocks
+- Implement proper error logging and monitoring
+- Handle both operational and programming errors separately
+
+### Performance
+- Use batch operations for high-throughput scenarios
+- Configure appropriate batch sizes and compression
+- Monitor and tune consumer group performance
+
+### Resource Management
+- Implement proper shutdown procedures
+- Clean up resources (disconnect producers/consumers)
+- Handle process signals (SIGTERM, SIGINT)
+
+### Message Processing
+- Validate message formats before processing
+- Implement proper serialization/deserialization
+- Handle message ordering when required
+
+## Contributing
+
+1. Fork the repository
+2. Create your feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -m 'Add some amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+---
+
+<div align="center">
+
+[![Built with Rust](https://img.shields.io/badge/Built%20with-Rust-orange)](https://www.rust-lang.org/)
+
+</div>

package/index.d.ts

@@ -1,131 +1,68 @@
-/* tslint:disable */
-/* eslint-disable */
-
-/* auto-generated by NAPI-RS */
-
-export const enum ConsumerResult {
-  Ok = 'Ok',
-  Retry = 'Retry'
-}
-export interface KafkaConsumerConfiguration {
-  topic: string
-  groupId: string
-  retryStrategy?: RetryStrategy
-  offset?: OffsetModel
-  createTopic?: boolean
-  commitMode?: CommitMode
-  enableAutoCommit?: boolean
-  configuration?: Record<string, string>
-  fecthMetadataTimeout?: number
-}
-export interface RetryStrategy {
-  retries: number
-  retryTopic?: string
-  dqlTopic?: string
-  pauseConsumerDuration?: number
-  offset?: OffsetModel
-  configuration?: Record<string, string>
-}
-export const enum CommitMode {
-  Sync = 0,
-  Async = 1
-}
-export interface ConsumerConfiguration {
-  groupId: string
-  createTopic?: boolean
-  enableAutoCommit?: boolean
-  configuration?: Record<string, string>
-  fecthMetadataTimeout?: number
-}
-export const enum PartitionPosition {
-  Beginning = 'Beginning',
-  End = 'End',
-  Stored = 'Stored'
-}
-export interface OffsetModel {
-  offset?: number
-  position?: PartitionPosition
-}
-export interface PartitionOffset {
-  partition: number
-  offset: OffsetModel
-}
-export interface TopicPartitionConfig {
-  topic: string
-  allOffsets?: OffsetModel
-  partitionOffset?: Array<PartitionOffset>
-}
-export const enum SecurityProtocol {
-  Plaintext = 'Plaintext',
-  Ssl = 'Ssl',
-  SaslPlaintext = 'SaslPlaintext',
-  SaslSsl = 'SaslSsl'
-}
-export interface KafkaConfiguration {
-  brokers: string
-  clientId: string
-  securityProtocol?: SecurityProtocol
-  configuration?: Record<string, string>
-  logLevel?: string
-  brokerAddressFamily?: string
-}
-export interface ProducerConfiguration {
-  queueTimeout?: number
-  thrownOnError?: boolean
-  configuration?: Record<string, string>
-}
-export interface Payload {
-  value: Buffer
-  key?: Buffer
-  headers?: Record<string, Buffer>
-  topic: string
-  partition: number
-  offset: number
-}
-export interface Message {
-  payload: Buffer
-  key?: Buffer
-  headers?: Record<string, Buffer>
-  topic: string
-  partition: number
-  offset: number
-}
-export interface RecordMetadata {
-  topic: string
-  partition: number
-  offset: number
-  error?: KafkaCrabError
-}
-export interface MessageProducer {
-  payload: Buffer
-  key?: Buffer
-  headers?: Record<string, Buffer>
-}
-export interface ProducerRecord {
-  topic: string
-  messages: Array<MessageProducer>
-}
-export interface KafkaCrabError {
-  code: number
-  message: string
-}
-export class KafkaConsumer {
-  startConsumer(callback: (error: Error | undefined, result: Payload) => Promise<ConsumerResult | undefined>): Promise<void>
-}
-export class KafkaStreamConsumer {
-  subscribe(topicConfigs: string | Array<TopicPartitionConfig>): Promise<void>
-  unsubscribe(): void
-  seek(topic: string, partition: number, offsetModel: OffsetModel, timeout?: number | undefined | null): void
-  recv(): Promise<Message>
-  commit(topic: string, partition: number, offset: number, commit: CommitMode): void
-}
+/**
+ * KafkaClient class
+ */
 export class KafkaClient {
-  readonly kafkaConfiguration: KafkaConfiguration
-  constructor(kafkaConfiguration: KafkaConfiguration)
-  createProducer(producerConfiguration: ProducerConfiguration): KafkaProducer
-  createConsumer(consumerConfiguration: KafkaConsumerConfiguration): KafkaConsumer
-  createStreamConsumer(consumerConfiguration: ConsumerConfiguration): KafkaStreamConsumer
-}
-export class KafkaProducer {
-  send(producerRecord: ProducerRecord): Promise<Array<RecordMetadata>>
-}
+    /**
+     * Creates a KafkaStreamReadable instance
+     * @param { KafkaConfiguration } config
+     */
+    constructor(config: KafkaConfiguration);
+    kafkaConfiguration: KafkaConfiguration;
+    kafkaClientConfig: KafkaClientConfig;
+    /**
+     * Creates a KafkaProducer instance
+     * @param { ProducerConfiguration | undefined } producerConfiguration
+     * @returns {KafkaProducer}
+     */
+    createProducer(producerConfiguration: ProducerConfiguration | undefined): KafkaProducer;
+    /**
+     * Creates a KafkaConsumer instance
+     * @param {ConsumerConfiguration } consumerConfiguration
+     * @returns {KafkaConsumer}
+     */
+    createConsumer(consumerConfiguration: ConsumerConfiguration): KafkaConsumer;
+    /**
+     * Creates a KafkaStreamReadable instance
+     * @param { ConsumerConfiguration } consumerConfiguration
+     * @returns {KafkaStreamReadable}
+     */
+    createStreamConsumer(consumerConfiguration: ConsumerConfiguration): KafkaStreamReadable;
+}
+/**
+ * KafkaStreamReadable class
+ * @extends Readable
+ */
+export class KafkaStreamReadable extends Readable {
+    /**
+     * Creates a KafkaStreamReadable instance
+     * @param { KafkaConsumer } kafkaConsumer
+     */
+    constructor(kafkaConsumer: KafkaConsumer);
+    kafkaConsumer: KafkaConsumer;
+    /**
+     * Subscribes to topics
+     * @param {string | Array<TopicPartitionConfig>} topics
+     * @returns
+     */
+    subscribe(topics: string | Array<TopicPartitionConfig>): Promise<void>;
+    /**
+     * Unsubscribe from topics
+     */
+    unsubscribe(): void;
+    /**
+     * The internal method called by the Readable stream to fetch data
+     */
+    _read(): Promise<void>;
+}
+import { ProducerConfiguration } from "./js-binding";
+import { ConsumerConfiguration } from "./js-binding";
+import { KafkaConfiguration } from "./js-binding";
+import { KafkaConsumer } from "./js-binding";
+import { CommitMode } from "./js-binding";
+import { PartitionPosition } from "./js-binding";
+import { SecurityProtocol } from "./js-binding";
+import { KafkaClientConfig } from "./js-binding";
+import { TopicPartitionConfig } from "./js-binding";
+import { KafkaProducer } from "./js-binding";
+import { Readable } from "stream";
+export { ProducerConfiguration, ConsumerConfiguration, KafkaConfiguration, KafkaConsumer, CommitMode, PartitionPosition, SecurityProtocol, KafkaClientConfig, TopicPartitionConfig, KafkaProducer };