@woovi/kafka 0.1.0

package/README.md

@@ -0,0 +1,421 @@
+# @woovi/kafka
+
+Kafka setup and utilities for Woovi microservices. Production-ready with built-in logging, metrics, graceful shutdown, and test utilities.
+
+## Installation
+
+```bash
+pnpm add @woovi/kafka
+```
+
+## Quick Start
+
+```typescript
+import {
+  createKafkaFromEnv,
+  createProducer,
+  createConsumer,
+  parseMessage,
+} from '@woovi/kafka';
+
+// Initialize Kafka client
+createKafkaFromEnv('my-service');
+
+// Create producer and publish
+const producer = createProducer({ defaultTopic: 'user-events' });
+await producer.publish({ data: { type: 'user.created', userId: '123' } });
+
+// Create consumer and process messages
+const consumer = createConsumer({
+  groupId: 'my-service-group',
+  topics: ['user-events'],
+});
+
+await consumer.run(async (payload) => {
+  const data = parseMessage<{ type: string; userId: string }>(payload);
+  console.log('Received:', data);
+});
+```
+
+## Configuration
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `KAFKA_BROKERS` | Comma-separated broker list | `localhost:9092` |
+| `KAFKA_SSL` | Enable SSL (`true`/`false`) | `false` |
+| `KAFKA_SASL_MECHANISM` | SASL mechanism | `plain` |
+| `KAFKA_SASL_USERNAME` | SASL username | - |
+| `KAFKA_SASL_PASSWORD` | SASL password | - |
+
+### Manual Configuration
+
+```typescript
+import { createKafka } from '@woovi/kafka';
+
+createKafka({
+  clientId: 'my-service',
+  brokers: ['kafka-1:9092', 'kafka-2:9092'],
+  ssl: true,
+  sasl: {
+    mechanism: 'scram-sha-512',
+    username: 'user',
+    password: 'password',
+  },
+  connectionTimeout: 10000,
+  requestTimeout: 30000,
+});
+```
+
+## Producer
+
+### Basic Usage
+
+```typescript
+import { createProducer } from '@woovi/kafka';
+
+const producer = createProducer({ defaultTopic: 'events' });
+
+// Publish single message
+await producer.publish({
+  data: { type: 'order.created', orderId: '456' },
+});
+
+// Publish with options
+await producer.publish({
+  data: { type: 'order.created', orderId: '456' },
+  key: 'order-456',        // Message key for partitioning
+  headers: { source: 'api' },
+  partition: 0,            // Specific partition (optional)
+});
+
+// Publish to different topic
+await producer.publish({
+  data: { type: 'notification' },
+  topic: 'notifications',
+});
+```
+
+### Batch Publishing
+
+```typescript
+// Batch to same topic
+await producer.publishBatch({
+  topic: 'events',
+  messages: [
+    { data: { type: 'event1' }, key: 'key1' },
+    { data: { type: 'event2' }, key: 'key2' },
+  ],
+});
+
+// Batch to multiple topics
+await producer.publishToMultipleTopics({
+  messages: [
+    { topic: 'events', data: { type: 'event1' } },
+    { topic: 'notifications', data: { type: 'notify' } },
+  ],
+});
+```
+
+### Producer Configuration
+
+```typescript
+const producer = createProducer({
+  defaultTopic: 'events',
+  idempotent: true,           // Enabled by default
+  maxInFlightRequests: 5,     // Max concurrent requests
+  transactionTimeout: 30000,  // Transaction timeout
+});
+```
+
+## Consumer
+
+### Basic Usage
+
+```typescript
+import { createConsumer, parseMessage, getMessageHeaders } from '@woovi/kafka';
+
+const consumer = createConsumer({
+  groupId: 'my-service-group',
+  topics: ['events', 'notifications'],
+  fromBeginning: false,  // Start from latest (default)
+});
+
+await consumer.run(async (payload) => {
+  const data = parseMessage<MyEventType>(payload);
+  const headers = getMessageHeaders(payload);
+
+  if (!data) {
+    console.error('Failed to parse message');
+    return;
+  }
+
+  console.log('Processing:', data, 'Headers:', headers);
+});
+```
+
+### Batch Processing
+
+```typescript
+await consumer.runBatch(async (payload) => {
+  for (const message of payload.batch.messages) {
+    const data = JSON.parse(message.value?.toString() || '{}');
+    // Process each message
+  }
+});
+```
+
+### Error Handling
+
+```typescript
+const consumer = createConsumer({
+  groupId: 'my-service-group',
+  topics: ['events'],
+  onError: async (error, payload) => {
+    // Send to error tracking (Sentry, etc.)
+    await sendToSentry(error, {
+      topic: payload.topic,
+      partition: payload.partition,
+      offset: payload.message.offset,
+    });
+
+    // Or send to dead letter queue
+    await dlqProducer.publish({
+      originalTopic: payload.topic,
+      error: error.message,
+      payload: payload.message.value?.toString(),
+    });
+  },
+});
+```
+
+### Consumer Configuration
+
+```typescript
+const consumer = createConsumer({
+  groupId: 'my-service-group',
+  topics: ['events'],
+  fromBeginning: false,
+  sessionTimeout: 30000,       // Session timeout (default: 30s)
+  rebalanceTimeout: 60000,     // Rebalance timeout (default: 60s)
+  heartbeatInterval: 3000,     // Heartbeat interval (default: 3s)
+  maxBytesPerPartition: 1048576, // Max bytes per partition
+  retry: {
+    initialRetryTime: 100,
+    retries: 8,
+  },
+});
+```
+
+### Pause and Resume
+
+```typescript
+// Pause all subscribed topics
+await consumer.pause();
+
+// Pause specific topics
+await consumer.pause(['events']);
+
+// Resume
+await consumer.resume();
+await consumer.resume(['events']);
+```
+
+## Health Check
+
+```typescript
+import { healthCheck } from '@woovi/kafka';
+
+// For load balancer / kubernetes probes
+app.get('/health', async (req, res) => {
+  const { healthy, error } = await healthCheck();
+
+  if (healthy) {
+    res.status(200).json({ status: 'ok' });
+  } else {
+    res.status(503).json({ status: 'error', error });
+  }
+});
+```
+
+## Graceful Shutdown
+
+Graceful shutdown is automatic on `SIGTERM` and `SIGINT`. All producers and consumers will disconnect cleanly.
+
+```typescript
+import { onShutdown } from '@woovi/kafka';
+
+// Add custom shutdown logic
+onShutdown(async () => {
+  console.log('Custom cleanup before Kafka disconnects');
+  await myCleanupFunction();
+});
+```
+
+## Prometheus Metrics
+
+### Setup
+
+```typescript
+import { getMetrics, getMetricsContentType, enableDefaultMetrics } from '@woovi/kafka';
+
+// Optional: enable Node.js default metrics
+enableDefaultMetrics();
+
+// Expose /metrics endpoint
+app.get('/metrics', async (req, res) => {
+  res.set('Content-Type', getMetricsContentType());
+  res.send(await getMetrics());
+});
+```
+
+### Available Metrics
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| `kafka_consumer_message_processing_duration_seconds` | Histogram | Message processing latency |
+| `kafka_consumer_messages_processed_total` | Counter | Total messages processed |
+| `kafka_consumer_messages_failed_total` | Counter | Failed messages |
+| `kafka_consumer_batch_processing_duration_seconds` | Histogram | Batch processing latency |
+| `kafka_consumer_batch_size` | Histogram | Messages per batch |
+| `kafka_consumer_last_message_timestamp_seconds` | Gauge | Last processed timestamp |
+| `kafka_producer_send_duration_seconds` | Histogram | Producer send latency |
+| `kafka_producer_messages_produced_total` | Counter | Total messages produced |
+
+### Grafana Queries
+
+```promql
+# P99 message processing latency
+histogram_quantile(0.99, rate(kafka_consumer_message_processing_duration_seconds_bucket[5m]))
+
+# Messages processed per second
+rate(kafka_consumer_messages_processed_total{status="success"}[1m])
+
+# Error rate percentage
+100 * rate(kafka_consumer_messages_failed_total[5m])
+  / rate(kafka_consumer_messages_processed_total[5m])
+
+# Producer latency P95
+histogram_quantile(0.95, rate(kafka_producer_send_duration_seconds_bucket[5m]))
+```
+
+## Testing
+
+### Setup Mocks
+
+```typescript
+// Jest
+jest.mock('kafkajs', () => require('@woovi/kafka/test-utils'));
+
+// Vitest
+vi.mock('kafkajs', () => import('@woovi/kafka/test-utils'));
+```
+
+### Test Utilities
+
+```typescript
+import {
+  kafkaAssert,
+  kafkaAssertLength,
+  getKafkaMessages,
+  clearAllMocks,
+} from '@woovi/kafka/test-utils';
+
+describe('MyService', () => {
+  beforeEach(() => {
+    clearAllMocks();
+  });
+
+  it('should publish user created event', async () => {
+    await myService.createUser({ name: 'John' });
+
+    // Assert message was published
+    kafkaAssert({
+      topic: 'user-events',
+      message: { type: 'user.created', name: 'John' },
+    });
+
+    // Assert message count
+    kafkaAssertLength({ topic: 'user-events', length: 1 });
+  });
+
+  it('should access raw messages', async () => {
+    await myService.createUser({ name: 'John' });
+
+    const messages = getKafkaMessages();
+    expect(messages[0].topic).toBe('user-events');
+  });
+});
+```
+
+### Available Test Exports
+
+```typescript
+import {
+  // Assertions
+  kafkaAssert,
+  kafkaAssertLength,
+  getKafkaMessages,
+  clearAllMocks,
+
+  // Mocks
+  Kafka,
+  mockProducer,
+  mockConsumer,
+  mockAdmin,
+  mockProducerSend,
+  mockTransactionSend,
+
+  // Error classes
+  KafkaJSNonRetriableError,
+  KafkaJSProtocolError,
+
+  // Constants
+  logLevel,
+  CompressionTypes,
+} from '@woovi/kafka/test-utils';
+```
+
+## API Reference
+
+### Producer Methods
+
+| Method | Description |
+|--------|-------------|
+| `connect()` | Connect to Kafka (auto-called on publish) |
+| `disconnect()` | Disconnect from Kafka |
+| `isConnected()` | Check connection status |
+| `publish(args)` | Publish single message |
+| `publishBatch(args)` | Publish multiple messages to one topic |
+| `publishToMultipleTopics(args)` | Publish to multiple topics |
+| `getProducer()` | Get underlying kafkajs producer |
+
+### Consumer Methods
+
+| Method | Description |
+|--------|-------------|
+| `connect()` | Connect to Kafka |
+| `subscribe()` | Subscribe to topics |
+| `disconnect()` | Disconnect from Kafka |
+| `isConnected()` | Check connection status |
+| `run(handler)` | Process messages one by one |
+| `runBatch(handler)` | Process messages in batches |
+| `pause(topics?)` | Pause consumption |
+| `resume(topics?)` | Resume consumption |
+| `getConsumer()` | Get underlying kafkajs consumer |
+
+### Utility Functions
+
+| Function | Description |
+|----------|-------------|
+| `parseMessage<T>(payload)` | Parse message value as JSON |
+| `getMessageHeaders(payload)` | Extract message headers |
+| `healthCheck()` | Check Kafka connectivity |
+| `onShutdown(callback)` | Register shutdown callback |
+| `getMetrics()` | Get Prometheus metrics string |
+| `getMetricsContentType()` | Get metrics content type |
+
+## License
+
+ISC

package/dist/consumer.d.ts

@@ -0,0 +1,25 @@
+import type { Consumer, EachMessagePayload } from 'kafkajs';
+import type { WooviConsumerConfig, MessageHandler, BatchHandler } from './types.js';
+export declare class WooviConsumer {
+    private consumer;
+    private topics;
+    private fromBeginning;
+    private connected;
+    private subscribed;
+    private groupId;
+    private onError?;
+    constructor(config: WooviConsumerConfig);
+    connect(): Promise<void>;
+    subscribe(): Promise<void>;
+    disconnect(): Promise<void>;
+    isConnected(): boolean;
+    run(handler: MessageHandler): Promise<void>;
+    runBatch(handler: BatchHandler): Promise<void>;
+    pause(topics?: string[]): Promise<void>;
+    resume(topics?: string[]): Promise<void>;
+    getConsumer(): Consumer;
+}
+export declare function createConsumer(config: WooviConsumerConfig): WooviConsumer;
+export declare function parseMessage<T>(payload: EachMessagePayload): T | null;
+export declare function getMessageHeaders(payload: EachMessagePayload): Record<string, string>;
+//# sourceMappingURL=consumer.d.ts.map

package/dist/consumer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAoB,MAAM,SAAS,CAAC;AAW9E,OAAO,KAAK,EAAE,mBAAmB,EAAE,cAAc,EAAE,YAAY,EAAgB,MAAM,YAAY,CAAC;AAIlG,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,MAAM,CAAgC;IAC9C,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAC,CAAe;gBAEnB,MAAM,EAAE,mBAAmB;IA6BjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAOxB,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAW1B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAQjC,WAAW,IAAI,OAAO;IAIhB,GAAG,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAoD3C,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C9C,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IASvC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9C,WAAW,IAAI,QAAQ;CAGxB;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,mBAAmB,GAAG,aAAa,CAEzE;AAED,wBAAgB,YAAY,CAAC,CAAC,EAAE,OAAO,EAAE,kBAAkB,GAAG,CAAC,GAAG,IAAI,CAuBrE;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAUrF"}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { createKafka, createKafkaFromEnv, getKafka, onShutdown, healthCheck, Kafka, logLevel } from './kafka.js';
+export { WooviProducer, createProducer } from './producer.js';
+export { WooviConsumer, createConsumer, parseMessage, getMessageHeaders, } from './consumer.js';
+export { kafkaRegistry, getMetrics, getMetricsContentType, resetMetrics, enableDefaultMetrics, messageProcessingDuration, messagesProcessedTotal, messagesFailedTotal, batchProcessingDuration, batchSize, consumerLag, lastMessageTimestamp, messagesProducedTotal, produceLatency, } from './metrics.js';
+export type { KafkaConfig, WooviProducerConfig, WooviConsumerConfig, PublishArgs, PublishBatchArgs, PublishMultiTopicArgs, MessageHandler, BatchHandler, ErrorHandler, Message, RecordMetadata, EachMessagePayload, EachBatchPayload, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAGjH,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAG9D,OAAO,EACL,aAAa,EACb,cAAc,EACd,YAAY,EACZ,iBAAiB,GAClB,MAAM,eAAe,CAAC;AAGvB,OAAO,EACL,aAAa,EACb,UAAU,EACV,qBAAqB,EACrB,YAAY,EACZ,oBAAoB,EAEpB,yBAAyB,EACzB,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,EACvB,SAAS,EACT,WAAW,EACX,oBAAoB,EACpB,qBAAqB,EACrB,cAAc,GACf,MAAM,cAAc,CAAC;AAGtB,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,YAAY,EACZ,OAAO,EACP,cAAc,EACd,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,YAAY,CAAC"}