kafka-crab-js 4.0.0-beta.2 → 4.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +573 -671
- package/assets/consumer-benchmark-snapshot.svg +189 -0
- package/dist/kafka-client.cjs +18 -83
- package/dist/kafka-client.cjs.map +1 -1
- package/dist/kafka-client.js +18 -83
- package/dist/kafka-client.js.map +1 -1
- package/dist/streams/base-kafka-stream-readable.cjs.map +1 -1
- package/dist/streams/base-kafka-stream-readable.js.map +1 -1
- package/dist/streams/kafka-batch-stream-readable.cjs.map +1 -1
- package/dist/streams/kafka-batch-stream-readable.js.map +1 -1
- package/dist/streams/kafka-stream-readable.cjs.map +1 -1
- package/dist/streams/kafka-stream-readable.js.map +1 -1
- package/js-binding.cjs +52 -52
- package/js-binding.d.ts +0 -4
- package/js-binding.js +52 -52
- package/package.json +8 -7
package/README.md
CHANGED
|
@@ -1,907 +1,809 @@
|
|
|
1
|
-
|
|
1
|
+
# Kafka Crab JS
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
A lightweight, flexible, and reliable Kafka client for JavaScript/TypeScript. It is built using Rust and librdkafka, providing a high-performance and feature-rich Kafka client.
|
|
3
|
+
A high-performance Kafka client for Node.js and TypeScript, implemented with Rust, NAPI-RS, librdkafka, and a small
|
|
4
|
+
JavaScript API layer.
|
|
6
5
|
|
|
7
6
|
[](https://www.npmjs.com/package/kafka-crab-js)
|
|
8
7
|
[](https://opensource.org/licenses/MIT)
|
|
9
8
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
---
|
|
9
|
+
## Highlights
|
|
13
10
|
|
|
14
|
-
|
|
11
|
+
- Native librdkafka client exposed through a TypeScript-friendly API.
|
|
12
|
+
- Producer, direct consumer, Node.js `Readable` stream, and native Web Stream consumer APIs.
|
|
13
|
+
- High-throughput batch receive APIs for workloads that can process more than one message at a time.
|
|
14
|
+
- Manual commit helpers that commit `message.offset + 1` correctly.
|
|
15
|
+
- Optional diagnostics-channel instrumentation for OpenTelemetry through `kafka-crab-js-otel`.
|
|
16
|
+
- Advanced librdkafka settings are forwarded through `configuration` without forcing a custom allowlist.
|
|
15
17
|
|
|
16
|
-
|
|
18
|
+
## Requirements
|
|
17
19
|
|
|
18
|
-
|
|
20
|
+
- Node.js `>= 22`.
|
|
21
|
+
- A Kafka broker reachable from the Node.js process.
|
|
22
|
+
- No separate librdkafka install is required for the published binaries.
|
|
19
23
|
|
|
20
|
-
|
|
24
|
+
## Table Of Contents
|
|
21
25
|
|
|
22
|
-
|
|
26
|
+
1. [Installation](#installation)
|
|
27
|
+
2. [Quick Start](#quick-start)
|
|
28
|
+
3. [Choosing A Consumer API](#choosing-a-consumer-api)
|
|
29
|
+
4. [Producer API](#producer-api)
|
|
30
|
+
5. [Consumer API](#consumer-api)
|
|
31
|
+
6. [Stream Consumers](#stream-consumers)
|
|
32
|
+
7. [Configuration](#configuration)
|
|
33
|
+
8. [OpenTelemetry](#opentelemetry)
|
|
34
|
+
9. [Performance Benchmarks](#performance-benchmarks)
|
|
35
|
+
10. [Migration Notes](#migration-notes)
|
|
36
|
+
11. [Troubleshooting](#troubleshooting)
|
|
37
|
+
12. [Development](#development)
|
|
38
|
+
13. [License](#license)
|
|
23
39
|
|
|
24
|
-
|
|
40
|
+
## Installation
|
|
25
41
|
|
|
26
|
-
```
|
|
27
|
-
|
|
42
|
+
```bash
|
|
43
|
+
npm install kafka-crab-js
|
|
28
44
|
```
|
|
29
45
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
```ts
|
|
33
|
-
import type { CommitMode, SecurityProtocol } from 'kafka-crab-js'
|
|
46
|
+
```bash
|
|
47
|
+
pnpm add kafka-crab-js
|
|
34
48
|
```
|
|
35
49
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
```ts
|
|
39
|
-
const commitMode: 'Sync' | 'Async' = 'Sync'
|
|
40
|
-
const securityProtocol: 'Plaintext' | 'Ssl' | 'SaslPlaintext' | 'SaslSsl' = 'Plaintext'
|
|
41
|
-
const eventName: 'PreRebalance' | 'PostRebalance' | 'CommitCallback' = 'PreRebalance'
|
|
42
|
-
const partitionPosition: 'Beginning' | 'End' | 'Stored' | 'Invalid' = 'Beginning'
|
|
50
|
+
```bash
|
|
51
|
+
yarn add kafka-crab-js
|
|
43
52
|
```
|
|
44
53
|
|
|
45
|
-
|
|
54
|
+
## Quick Start
|
|
46
55
|
|
|
47
|
-
|
|
48
|
-
| ------------------- | ------------------------------------------------------- |
|
|
49
|
-
| `CommitMode` | `'Sync' \| 'Async'` |
|
|
50
|
-
| `KafkaEventName` | `'PreRebalance' \| 'PostRebalance' \| 'CommitCallback'` |
|
|
51
|
-
| `PartitionPosition` | `'Beginning' \| 'End' \| 'Stored' \| 'Invalid'` |
|
|
52
|
-
| `SecurityProtocol` | `'Plaintext' \| 'Ssl' \| 'SaslPlaintext' \| 'SaslSsl'` |
|
|
56
|
+
### Produce Messages
|
|
53
57
|
|
|
54
|
-
|
|
58
|
+
```ts
|
|
59
|
+
import { KafkaClient } from 'kafka-crab-js'
|
|
55
60
|
|
|
56
|
-
|
|
61
|
+
const client = new KafkaClient({
|
|
62
|
+
brokers: 'localhost:9092',
|
|
63
|
+
clientId: 'orders-api',
|
|
64
|
+
securityProtocol: 'Plaintext',
|
|
65
|
+
})
|
|
57
66
|
|
|
58
|
-
|
|
67
|
+
const producer = client.createProducer()
|
|
59
68
|
|
|
60
|
-
|
|
69
|
+
const metadata = await producer.send({
|
|
70
|
+
topic: 'orders',
|
|
71
|
+
messages: [
|
|
72
|
+
{
|
|
73
|
+
key: Buffer.from('order-123'),
|
|
74
|
+
payload: Buffer.from(JSON.stringify({ id: 'order-123', status: 'created' })),
|
|
75
|
+
headers: {
|
|
76
|
+
'content-type': Buffer.from('application/json'),
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
],
|
|
80
|
+
})
|
|
61
81
|
|
|
62
|
-
|
|
82
|
+
console.log(metadata)
|
|
83
|
+
```
|
|
63
84
|
|
|
64
|
-
|
|
85
|
+
### Consume One Message At A Time
|
|
65
86
|
|
|
66
|
-
```
|
|
87
|
+
```ts
|
|
67
88
|
import { KafkaClient } from 'kafka-crab-js'
|
|
68
89
|
|
|
69
90
|
const client = new KafkaClient({
|
|
70
91
|
brokers: 'localhost:9092',
|
|
71
|
-
clientId: '
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
92
|
+
clientId: 'orders-worker',
|
|
93
|
+
securityProtocol: 'Plaintext',
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
const consumer = client.createConsumer({
|
|
97
|
+
groupId: 'orders-worker',
|
|
98
|
+
enableAutoCommit: false,
|
|
99
|
+
configuration: {
|
|
100
|
+
'auto.offset.reset': 'earliest',
|
|
75
101
|
},
|
|
76
102
|
})
|
|
77
|
-
```
|
|
78
103
|
|
|
79
|
-
|
|
104
|
+
await consumer.subscribe('orders')
|
|
80
105
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
106
|
+
try {
|
|
107
|
+
while (true) {
|
|
108
|
+
const message = await consumer.recv()
|
|
109
|
+
if (!message) {
|
|
110
|
+
break
|
|
111
|
+
}
|
|
84
112
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
enableOtelInstrumentation({
|
|
88
|
-
metrics: { enabled: true },
|
|
89
|
-
})
|
|
113
|
+
const value = JSON.parse(message.payload.toString('utf8'))
|
|
114
|
+
console.log({ value, topic: message.topic, partition: message.partition, offset: message.offset })
|
|
90
115
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
116
|
+
await consumer.commitMessage(message, 'Sync')
|
|
117
|
+
}
|
|
118
|
+
} finally {
|
|
119
|
+
await consumer.disconnect()
|
|
120
|
+
}
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### Consume In Batches
|
|
124
|
+
|
|
125
|
+
```ts
|
|
126
|
+
const consumer = client.createConsumer({
|
|
127
|
+
groupId: 'orders-batch-worker',
|
|
128
|
+
enableAutoCommit: false,
|
|
129
|
+
configuration: {
|
|
130
|
+
'auto.offset.reset': 'earliest',
|
|
131
|
+
},
|
|
96
132
|
})
|
|
97
133
|
|
|
98
|
-
|
|
99
|
-
const message = await consumer.recv()
|
|
100
|
-
// ... process message ...
|
|
101
|
-
endSpan(message)
|
|
102
|
-
```
|
|
134
|
+
await consumer.subscribe('orders')
|
|
103
135
|
|
|
104
|
-
|
|
136
|
+
try {
|
|
137
|
+
while (true) {
|
|
138
|
+
const batch = await consumer.recvBatch(500, 50)
|
|
139
|
+
if (batch.length === 0) {
|
|
140
|
+
continue
|
|
141
|
+
}
|
|
105
142
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
143
|
+
for (const message of batch) {
|
|
144
|
+
await processOrder(message.payload)
|
|
145
|
+
await consumer.commitMessage(message, 'Async')
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
} finally {
|
|
149
|
+
await consumer.disconnect()
|
|
150
|
+
}
|
|
151
|
+
```
|
|
112
152
|
|
|
113
|
-
|
|
153
|
+
## Choosing A Consumer API
|
|
114
154
|
|
|
115
|
-
|
|
|
116
|
-
|
|
|
117
|
-
| `
|
|
118
|
-
| `
|
|
155
|
+
| API | Emits | Best For |
|
|
156
|
+
| ---------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------ |
|
|
157
|
+
| `consumer.recv()` | `Message \| null` | Simple sequential workers and explicit control. |
|
|
158
|
+
| `consumer.recvBatch()` | `Message[]` | High-throughput workers that can process chunks. |
|
|
159
|
+
| `consumer.recvStream()` | Web `ReadableStream<Message>` | Pull-based Web Stream workflows. |
|
|
160
|
+
| `consumer.recvBatchStream()` | Web `ReadableStream<Message[]>` | Native batch streaming with explicit consumer control. |
|
|
161
|
+
| `client.createStreamConsumer()` | Node.js `Readable` emitting `Message` | Existing Node stream pipelines. |
|
|
162
|
+
| `client.createWebStreamConsumer()` | Web `ReadableStream<Message>` or `ReadableStream<Message[]>` | v4 high-throughput stream consumers. |
|
|
119
163
|
|
|
120
|
-
|
|
164
|
+
Two details matter for choosing correctly:
|
|
121
165
|
|
|
122
|
-
-
|
|
123
|
-
|
|
124
|
-
-
|
|
166
|
+
- `createStreamConsumer()` is a Node.js `Readable` compatibility API. Even when `batchSize > 1`, it emits individual
|
|
167
|
+
`Message` objects and uses batching internally to reduce native boundary crossings.
|
|
168
|
+
- `createWebStreamConsumer()` returns a discriminated object. In serial mode it emits `Message`; in batch mode it emits
|
|
169
|
+
`Message[]`. Use this API when your code can process whole batches directly.
|
|
125
170
|
|
|
126
|
-
|
|
171
|
+
## Producer API
|
|
127
172
|
|
|
128
|
-
|
|
173
|
+
Create a producer from a `KafkaClient`:
|
|
129
174
|
|
|
130
|
-
|
|
131
|
-
|
|
175
|
+
```ts
|
|
176
|
+
const producer = client.createProducer({
|
|
177
|
+
autoFlush: true,
|
|
178
|
+
queueTimeout: 5000,
|
|
179
|
+
configuration: {
|
|
180
|
+
'compression.type': 'lz4',
|
|
181
|
+
acks: 'all',
|
|
182
|
+
},
|
|
183
|
+
})
|
|
184
|
+
```
|
|
132
185
|
|
|
133
|
-
|
|
186
|
+
Send one or more messages:
|
|
134
187
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
- **After** (v2.1.0):
|
|
145
|
-
```javascript
|
|
146
|
-
const message = await consumer.recv()
|
|
147
|
-
await consumer.commitMessage(message, 'Sync')
|
|
148
|
-
```
|
|
188
|
+
```ts
|
|
189
|
+
const records = await producer.send({
|
|
190
|
+
topic: 'orders',
|
|
191
|
+
messages: [
|
|
192
|
+
{ key: Buffer.from('1'), payload: Buffer.from('first') },
|
|
193
|
+
{ key: Buffer.from('2'), payload: Buffer.from('second') },
|
|
194
|
+
],
|
|
195
|
+
})
|
|
196
|
+
```
|
|
149
197
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
- Safe handling when OTEL SDK is not installed (no-op behavior)
|
|
153
|
-
- Better span context management across producer and consumer operations
|
|
154
|
-
- Seamless integration with standard OTEL SDK setup
|
|
198
|
+
When `autoFlush` is enabled, `send()` waits for delivery confirmation and returns `RecordMetadata[]`. When `autoFlush`
|
|
199
|
+
is disabled, `send()` buffers messages and `flush()` sends pending messages:
|
|
155
200
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
- GitHub Actions updated to v6
|
|
159
|
-
- Improved caching with actions/cache v4
|
|
201
|
+
```ts
|
|
202
|
+
const producer = client.createProducer({ autoFlush: false })
|
|
160
203
|
|
|
161
|
-
|
|
204
|
+
await producer.send({
|
|
205
|
+
topic: 'orders',
|
|
206
|
+
messages: [{ payload: Buffer.from('buffered') }],
|
|
207
|
+
})
|
|
162
208
|
|
|
163
|
-
|
|
164
|
-
|
|
209
|
+
const records = await producer.flush()
|
|
210
|
+
console.log(records)
|
|
211
|
+
```
|
|
165
212
|
|
|
166
|
-
|
|
213
|
+
`producer.inFlightCount()` returns the number of messages sent but not yet acknowledged.
|
|
167
214
|
|
|
168
|
-
|
|
169
|
-
- **REMOVED**: `createTopic` field from `ConsumerConfiguration`
|
|
170
|
-
- **Migration**: Use `createTopic` field in `TopicPartitionConfig` instead when subscribing to topics
|
|
215
|
+
## Consumer API
|
|
171
216
|
|
|
172
|
-
|
|
173
|
-
- Stream consumers now properly implement Node.js stream lifecycle methods (`_destroy()`)
|
|
174
|
-
- **Memory leak prevention**: Streams now automatically disconnect Kafka consumers during destruction
|
|
217
|
+
### Subscribe
|
|
175
218
|
|
|
176
|
-
|
|
177
|
-
- **BREAKING**: The `consumer.commit()` method is now async and must be awaited
|
|
219
|
+
The simplest subscription accepts a topic string:
|
|
178
220
|
|
|
179
|
-
|
|
221
|
+
```ts
|
|
222
|
+
await consumer.subscribe('orders')
|
|
223
|
+
```
|
|
180
224
|
|
|
181
|
-
|
|
225
|
+
Use `TopicPartitionConfig[]` when you need offsets, partition-specific offsets, or topic creation:
|
|
182
226
|
|
|
183
|
-
|
|
227
|
+
```ts
|
|
228
|
+
await consumer.subscribe([
|
|
229
|
+
{
|
|
230
|
+
topic: 'orders',
|
|
231
|
+
createTopic: true,
|
|
232
|
+
numPartitions: 3,
|
|
233
|
+
replicas: 1,
|
|
234
|
+
allOffsets: { position: 'Beginning' },
|
|
235
|
+
},
|
|
236
|
+
])
|
|
237
|
+
```
|
|
184
238
|
|
|
185
|
-
-
|
|
186
|
-
- 🚀 High-performance message processing
|
|
187
|
-
- 🔄 Automatic reconnection handling
|
|
188
|
-
- 🎯 Type-safe interfaces (TypeScript support)
|
|
189
|
-
- ⚡ Async/await support
|
|
190
|
-
- 🛠️ Configurable consumer and producer options
|
|
191
|
-
- 📊 Stream processing support with configurable stream options
|
|
192
|
-
- 📦 Message batching capabilities
|
|
193
|
-
- 🔍 Comprehensive error handling
|
|
194
|
-
- 📈 Performance benchmarking suite
|
|
195
|
-
- 🔧 Flexible configuration system supporting complex data types
|
|
239
|
+
For partition-specific offsets:
|
|
196
240
|
|
|
197
|
-
|
|
241
|
+
```ts
|
|
242
|
+
await consumer.subscribe([
|
|
243
|
+
{
|
|
244
|
+
topic: 'orders',
|
|
245
|
+
partitionOffset: [
|
|
246
|
+
{
|
|
247
|
+
partition: 0,
|
|
248
|
+
offset: { offset: 42 },
|
|
249
|
+
},
|
|
250
|
+
],
|
|
251
|
+
},
|
|
252
|
+
])
|
|
253
|
+
```
|
|
198
254
|
|
|
199
|
-
|
|
200
|
-
2. [Quick Start](#quick-start)
|
|
201
|
-
3. [Consumer Examples](#basic-consumer-setup)
|
|
202
|
-
4. [Producer Examples](#producer-examples)
|
|
203
|
-
5. [Stream Processing](#stream-processing)
|
|
204
|
-
6. [Configuration](#configuration)
|
|
205
|
-
7. [Performance Benchmarks](#performance-benchmarks)
|
|
206
|
-
8. [Best Practices](#best-practices)
|
|
207
|
-
9. [Contributing](#contributing)
|
|
208
|
-
10. [License](#license)
|
|
209
|
-
11. [OpenTelemetry Instrumentation](#opentelemetry-instrumentation)
|
|
255
|
+
### Receive
|
|
210
256
|
|
|
211
|
-
|
|
257
|
+
`recv()` waits for one message and returns `null` when the consumer is disconnected:
|
|
212
258
|
|
|
213
|
-
```
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
pnpm add kafka-crab-js
|
|
259
|
+
```ts
|
|
260
|
+
const message = await consumer.recv()
|
|
261
|
+
if (message) {
|
|
262
|
+
await handleMessage(message)
|
|
263
|
+
}
|
|
219
264
|
```
|
|
220
265
|
|
|
221
|
-
|
|
266
|
+
`recvBatch(size, timeoutMs)` returns up to `size` messages:
|
|
222
267
|
|
|
223
|
-
|
|
268
|
+
```ts
|
|
269
|
+
const messages = await consumer.recvBatch(1000, 100)
|
|
270
|
+
```
|
|
224
271
|
|
|
225
|
-
|
|
226
|
-
import { KafkaClient } from 'kafka-crab-js'
|
|
227
|
-
async function run() {
|
|
228
|
-
const kafkaClient = new KafkaClient({
|
|
229
|
-
brokers: 'localhost:29092',
|
|
230
|
-
clientId: 'foo-client',
|
|
231
|
-
logLevel: 'debug',
|
|
232
|
-
brokerAddressFamily: 'v4',
|
|
233
|
-
})
|
|
234
|
-
|
|
235
|
-
// Create consumer with topic creation control
|
|
236
|
-
const consumer = kafkaClient.createConsumer({
|
|
237
|
-
groupId: 'foo-group',
|
|
238
|
-
configuration: {
|
|
239
|
-
'auto.offset.reset': 'earliest',
|
|
240
|
-
'enable.auto.commit': false, // Use manual commit for better control
|
|
241
|
-
},
|
|
242
|
-
})
|
|
272
|
+
An empty array means the timeout elapsed without a full batch. It does not mean the consumer is closed.
|
|
243
273
|
|
|
244
|
-
|
|
245
|
-
await consumer.subscribe([
|
|
246
|
-
{
|
|
247
|
-
topic: 'foo',
|
|
248
|
-
createTopic: true,
|
|
249
|
-
numPartitions: 3,
|
|
250
|
-
},
|
|
251
|
-
])
|
|
274
|
+
### Commit
|
|
252
275
|
|
|
253
|
-
|
|
254
|
-
const { payload, partition, offset, topic } = message
|
|
255
|
-
console.log({
|
|
256
|
-
topic,
|
|
257
|
-
partition,
|
|
258
|
-
offset,
|
|
259
|
-
value: payload.toString(),
|
|
260
|
-
})
|
|
276
|
+
Use `commitMessage()` for normal message processing. It commits `message.offset + 1`, which is the offset Kafka expects:
|
|
261
277
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
278
|
+
```ts
|
|
279
|
+
await consumer.commitMessage(message, 'Sync')
|
|
280
|
+
```
|
|
265
281
|
|
|
266
|
-
|
|
267
|
-
// await consumer.commit(topic, partition, offset + 1, 'Sync');
|
|
282
|
+
Use `commit()` when you already calculated the offset:
|
|
268
283
|
|
|
269
|
-
|
|
270
|
-
|
|
284
|
+
```ts
|
|
285
|
+
await consumer.commit(message.topic, message.partition, message.offset + 1, 'Async')
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
Commit modes are string literal values:
|
|
271
289
|
|
|
272
|
-
|
|
290
|
+
```ts
|
|
291
|
+
type CommitMode = 'Sync' | 'Async'
|
|
273
292
|
```
|
|
274
293
|
|
|
275
|
-
###
|
|
294
|
+
### Pause, Resume, Seek, And Assignment
|
|
276
295
|
|
|
277
|
-
```
|
|
278
|
-
|
|
296
|
+
```ts
|
|
297
|
+
consumer.pause()
|
|
298
|
+
consumer.resume()
|
|
279
299
|
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
clientId: 'my-client-id',
|
|
283
|
-
logLevel: 'info',
|
|
284
|
-
brokerAddressFamily: 'v4',
|
|
285
|
-
})
|
|
300
|
+
consumer.seek('orders', 0, { position: 'Beginning' })
|
|
301
|
+
consumer.seek('orders', 0, { offset: 42 })
|
|
286
302
|
|
|
287
|
-
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
'message.timeout.ms': 5000, // Now supports number values
|
|
291
|
-
'batch.size': 16384,
|
|
292
|
-
'compression.type': 'snappy',
|
|
293
|
-
},
|
|
294
|
-
})
|
|
303
|
+
const assignment = consumer.assignment()
|
|
304
|
+
const subscription = consumer.getSubscription()
|
|
305
|
+
```
|
|
295
306
|
|
|
296
|
-
|
|
297
|
-
id: 1,
|
|
298
|
-
name: 'Sample Message',
|
|
299
|
-
timestamp: new Date().toISOString(),
|
|
300
|
-
}
|
|
307
|
+
Consumer events are available through `onEvents()`:
|
|
301
308
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
+
```ts
|
|
310
|
+
consumer.onEvents((error, event) => {
|
|
311
|
+
if (error) {
|
|
312
|
+
console.error(error)
|
|
313
|
+
return
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
console.log(event.name, event.payload)
|
|
309
317
|
})
|
|
318
|
+
```
|
|
310
319
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
320
|
+
### Cleanup
|
|
321
|
+
|
|
322
|
+
Always disconnect direct consumers:
|
|
323
|
+
|
|
324
|
+
```ts
|
|
325
|
+
try {
|
|
326
|
+
await consumer.subscribe('orders')
|
|
327
|
+
// consume...
|
|
328
|
+
} finally {
|
|
329
|
+
await consumer.disconnect()
|
|
316
330
|
}
|
|
317
331
|
```
|
|
318
332
|
|
|
319
|
-
## Stream
|
|
333
|
+
## Stream Consumers
|
|
320
334
|
|
|
321
|
-
###
|
|
335
|
+
### Node.js Readable Stream
|
|
322
336
|
|
|
323
|
-
|
|
324
|
-
import { KafkaClient } from 'kafka-crab-js'
|
|
337
|
+
Use `createStreamConsumer()` when integrating with Node.js stream tooling. It emits `Message` objects.
|
|
325
338
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
const kafkaStream = kafkaClient.createStreamConsumer(
|
|
335
|
-
{
|
|
336
|
-
groupId: `my-group-id`,
|
|
337
|
-
enableAutoCommit: true,
|
|
338
|
-
},
|
|
339
|
-
{
|
|
340
|
-
objectMode: true, // Default in v2.0.0+
|
|
341
|
-
highWaterMark: 1024,
|
|
342
|
-
encoding: null,
|
|
339
|
+
```ts
|
|
340
|
+
const stream = client.createStreamConsumer({
|
|
341
|
+
groupId: 'orders-stream',
|
|
342
|
+
enableAutoCommit: false,
|
|
343
|
+
batchSize: 256,
|
|
344
|
+
batchTimeout: 50,
|
|
345
|
+
configuration: {
|
|
346
|
+
'auto.offset.reset': 'earliest',
|
|
343
347
|
},
|
|
344
|
-
)
|
|
345
|
-
|
|
346
|
-
await kafkaStream.subscribe([
|
|
347
|
-
{ topic: 'foo', createTopic: true },
|
|
348
|
-
{ topic: 'bar', createTopic: true },
|
|
349
|
-
])
|
|
348
|
+
})
|
|
350
349
|
|
|
351
|
-
|
|
352
|
-
console.log('>>> Message received:', {
|
|
353
|
-
payload: message.payload.toString(),
|
|
354
|
-
offset: message.offset,
|
|
355
|
-
partition: message.partition,
|
|
356
|
-
topic: message.topic,
|
|
357
|
-
})
|
|
350
|
+
await stream.subscribe('orders')
|
|
358
351
|
|
|
359
|
-
|
|
360
|
-
|
|
352
|
+
stream.on('data', async (message) => {
|
|
353
|
+
try {
|
|
354
|
+
await processOrder(message.payload)
|
|
355
|
+
await stream.commitMessage(message, 'Async')
|
|
356
|
+
} catch (error) {
|
|
357
|
+
stream.destroy(error instanceof Error ? error : new Error(String(error)))
|
|
361
358
|
}
|
|
362
359
|
})
|
|
363
360
|
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
361
|
+
stream.on('error', (error) => {
|
|
362
|
+
console.error('stream error', error)
|
|
363
|
+
})
|
|
364
|
+
|
|
365
|
+
stream.on('close', () => {
|
|
366
|
+
console.log('stream closed')
|
|
367
367
|
})
|
|
368
368
|
```
|
|
369
369
|
|
|
370
|
-
|
|
370
|
+
Destroy stream consumers instead of calling `disconnect()` directly. The stream cleanup path cancels the source reader,
|
|
371
|
+
unsubscribes, and disconnects the native consumer:
|
|
372
|
+
|
|
373
|
+
```ts
|
|
374
|
+
stream.destroy()
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
### Native Web Stream
|
|
371
378
|
|
|
372
|
-
|
|
379
|
+
Use `createWebStreamConsumer()` when you want a Web Stream and clear serial/batch typing.
|
|
373
380
|
|
|
374
|
-
|
|
375
|
-
const kafkaClient = new KafkaClient({
|
|
376
|
-
brokers: 'localhost:29092',
|
|
377
|
-
clientId: 'my-client-id',
|
|
378
|
-
brokerAddressFamily: 'v4',
|
|
379
|
-
})
|
|
381
|
+
Serial mode is selected when `batchSize` is omitted, `0`, or `1`:
|
|
380
382
|
|
|
381
|
-
|
|
382
|
-
const
|
|
383
|
+
```ts
|
|
384
|
+
const webConsumer = client.createWebStreamConsumer({
|
|
385
|
+
groupId: 'orders-web-serial',
|
|
386
|
+
serialPrefetchSize: 64,
|
|
387
|
+
serialPrefetchTimeout: 1,
|
|
383
388
|
configuration: {
|
|
384
|
-
'
|
|
385
|
-
'linger.ms': 10, // Number value supported
|
|
386
|
-
'compression.type': 'lz4',
|
|
387
|
-
'enable.idempotence': true, // Boolean value supported
|
|
389
|
+
'auto.offset.reset': 'earliest',
|
|
388
390
|
},
|
|
389
391
|
})
|
|
390
392
|
|
|
391
|
-
|
|
392
|
-
payload: Buffer.from(
|
|
393
|
-
JSON.stringify({
|
|
394
|
-
_id: i,
|
|
395
|
-
name: `Batch Message ${i}`,
|
|
396
|
-
timestamp: new Date().toISOString(),
|
|
397
|
-
}),
|
|
398
|
-
),
|
|
399
|
-
}))
|
|
400
|
-
|
|
401
|
-
try {
|
|
402
|
-
const result = await producer.send({
|
|
403
|
-
topic: 'my-topic',
|
|
404
|
-
messages,
|
|
405
|
-
})
|
|
406
|
-
console.log('Batch sent. Offset:', result)
|
|
407
|
-
console.assert(result.length === 100)
|
|
408
|
-
} catch (error) {
|
|
409
|
-
console.error('Batch error:', error)
|
|
410
|
-
}
|
|
411
|
-
```
|
|
393
|
+
await webConsumer.consumer.subscribe('orders')
|
|
412
394
|
|
|
413
|
-
|
|
395
|
+
const reader = webConsumer.stream.getReader()
|
|
414
396
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
retries: 5,
|
|
421
|
-
'max.in.flight.requests.per.connection': 1,
|
|
422
|
-
},
|
|
423
|
-
})
|
|
424
|
-
|
|
425
|
-
try {
|
|
426
|
-
await producer.send({
|
|
427
|
-
topic: 'user-events',
|
|
428
|
-
messages: [
|
|
429
|
-
{
|
|
430
|
-
key: 'user-123',
|
|
431
|
-
payload: Buffer.from(
|
|
432
|
-
JSON.stringify({
|
|
433
|
-
userId: 123,
|
|
434
|
-
action: 'update',
|
|
435
|
-
}),
|
|
436
|
-
),
|
|
437
|
-
headers: {
|
|
438
|
-
'correlation-id': 'txn-123',
|
|
439
|
-
source: 'user-service',
|
|
440
|
-
},
|
|
441
|
-
},
|
|
442
|
-
],
|
|
443
|
-
})
|
|
444
|
-
} catch (error) {
|
|
445
|
-
console.error('Error:', error)
|
|
397
|
+
try {
|
|
398
|
+
const { value: message, done } = await reader.read()
|
|
399
|
+
if (!done && message) {
|
|
400
|
+
await processOrder(message.payload)
|
|
401
|
+
await webConsumer.consumer.commitMessage(message, 'Async')
|
|
446
402
|
}
|
|
403
|
+
} finally {
|
|
404
|
+
await reader.cancel()
|
|
405
|
+
await webConsumer.consumer.disconnect()
|
|
447
406
|
}
|
|
448
407
|
```
|
|
449
408
|
|
|
450
|
-
|
|
409
|
+
Batch mode is selected with `batchSize > 1` and emits `Message[]` chunks:
|
|
451
410
|
|
|
452
|
-
```
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
clientId: 'reconnect-test',
|
|
458
|
-
logLevel: 'debug',
|
|
459
|
-
brokerAddressFamily: 'v4',
|
|
411
|
+
```ts
|
|
412
|
+
const webConsumer = client.createWebStreamConsumer({
|
|
413
|
+
groupId: 'orders-web-batch',
|
|
414
|
+
batchSize: 1024,
|
|
415
|
+
batchTimeout: 10,
|
|
460
416
|
configuration: {
|
|
461
417
|
'auto.offset.reset': 'earliest',
|
|
462
|
-
'session.timeout.ms': 30000,
|
|
463
|
-
'heartbeat.interval.ms': 10000,
|
|
464
418
|
},
|
|
465
419
|
})
|
|
466
420
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
async function createConsumer() {
|
|
471
|
-
const kafkaStream = kafkaClient.createStreamConsumer(
|
|
472
|
-
{
|
|
473
|
-
groupId: 'reconnect-test',
|
|
474
|
-
enableAutoCommit: true,
|
|
475
|
-
},
|
|
476
|
-
{
|
|
477
|
-
highWaterMark: 100,
|
|
478
|
-
objectMode: true,
|
|
479
|
-
},
|
|
480
|
-
)
|
|
481
|
-
|
|
482
|
-
await kafkaStream.subscribe([
|
|
483
|
-
{ topic: 'foo', createTopic: true },
|
|
484
|
-
{ topic: 'bar', createTopic: true },
|
|
485
|
-
])
|
|
486
|
-
return kafkaStream
|
|
487
|
-
}
|
|
421
|
+
if (webConsumer.mode === 'batch') {
|
|
422
|
+
await webConsumer.consumer.subscribe('orders')
|
|
423
|
+
const reader = webConsumer.stream.getReader()
|
|
488
424
|
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
const MAX_RETRIES = 5
|
|
496
|
-
const RETRY_DELAY = 5000 // 5 seconds
|
|
497
|
-
|
|
498
|
-
async function handleRetry() {
|
|
499
|
-
if (retryCount < MAX_RETRIES) {
|
|
500
|
-
retryCount++
|
|
501
|
-
console.log(
|
|
502
|
-
`Attempting to restart consumer (attempt ${retryCount}/${MAX_RETRIES}) in ${RETRY_DELAY / 1000} seconds...`,
|
|
503
|
-
)
|
|
504
|
-
setTimeout(setupConsumerWithRetry, RETRY_DELAY)
|
|
505
|
-
} else {
|
|
506
|
-
console.error(`Maximum retry attempts (${MAX_RETRIES}) reached. Stopping consumer.`)
|
|
507
|
-
process.exit(1)
|
|
425
|
+
try {
|
|
426
|
+
const { value: batch, done } = await reader.read()
|
|
427
|
+
if (!done && batch) {
|
|
428
|
+
for (const message of batch) {
|
|
429
|
+
await processOrder(message.payload)
|
|
430
|
+
}
|
|
508
431
|
}
|
|
432
|
+
} finally {
|
|
433
|
+
await reader.cancel()
|
|
434
|
+
await webConsumer.consumer.disconnect()
|
|
509
435
|
}
|
|
436
|
+
}
|
|
437
|
+
```
|
|
510
438
|
|
|
511
|
-
|
|
512
|
-
try {
|
|
513
|
-
const kafkaStream = await createConsumer()
|
|
514
|
-
retryCount = 0 // Reset retry count on successful connection
|
|
515
|
-
|
|
516
|
-
console.log('Starting consumer')
|
|
517
|
-
|
|
518
|
-
kafkaStream.on('data', (message) => {
|
|
519
|
-
counter++
|
|
520
|
-
console.log('>>> Message received:', {
|
|
521
|
-
counter,
|
|
522
|
-
payload: message.payload.toString(),
|
|
523
|
-
offset: message.offset,
|
|
524
|
-
partition: message.partition,
|
|
525
|
-
topic: message.topic,
|
|
526
|
-
})
|
|
527
|
-
})
|
|
528
|
-
|
|
529
|
-
kafkaStream.on('error', async (error) => {
|
|
530
|
-
console.error('Stream error:', error)
|
|
531
|
-
handleRetry()
|
|
532
|
-
})
|
|
533
|
-
|
|
534
|
-
kafkaStream.on('close', () => {
|
|
535
|
-
console.log('Stream ended')
|
|
536
|
-
try {
|
|
537
|
-
kafkaStream.unsubscribe()
|
|
538
|
-
} catch (unsubError) {
|
|
539
|
-
console.error('Error unsubscribing:', unsubError)
|
|
540
|
-
}
|
|
541
|
-
})
|
|
542
|
-
} catch (error) {
|
|
543
|
-
console.error('Error setting up consumer:', error)
|
|
544
|
-
handleRetry()
|
|
545
|
-
}
|
|
546
|
-
}
|
|
439
|
+
## Configuration
|
|
547
440
|
|
|
548
|
-
|
|
549
|
-
}
|
|
441
|
+
### KafkaClient
|
|
550
442
|
|
|
551
|
-
|
|
443
|
+
```ts
|
|
444
|
+
const client = new KafkaClient({
|
|
445
|
+
brokers: 'localhost:9092',
|
|
446
|
+
clientId: 'orders-service',
|
|
447
|
+
securityProtocol: 'Plaintext',
|
|
448
|
+
logLevel: 'info',
|
|
449
|
+
brokerAddressFamily: 'v4',
|
|
450
|
+
diagnostics: true,
|
|
451
|
+
configuration: {
|
|
452
|
+
'socket.keepalive.enable': true,
|
|
453
|
+
},
|
|
454
|
+
})
|
|
552
455
|
```
|
|
553
456
|
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
457
|
+
| Option | Type | Default | Description |
|
|
458
|
+
| --------------------- | ------------------------------------------------------ | --------- | ------------------------------------------------------------------------- |
|
|
459
|
+
| `brokers` | `string` | required | Comma-separated broker list, for example `localhost:9092,localhost:9093`. |
|
|
460
|
+
| `clientId` | `string` | `rdkafka` | Client identifier sent to Kafka. |
|
|
461
|
+
| `securityProtocol` | `'Plaintext' \| 'Ssl' \| 'SaslPlaintext' \| 'SaslSsl'` | | Security protocol. |
|
|
462
|
+
| `logLevel` | `string` | | librdkafka log level. |
|
|
463
|
+
| `brokerAddressFamily` | `string` | | Address family hint such as `v4`. |
|
|
464
|
+
| `diagnostics` | `boolean` | `true` | Enables diagnostic-channel events used by `kafka-crab-js-otel`. |
|
|
465
|
+
| `configuration` | `Record<string, any>` | | Additional librdkafka client settings. |
|
|
557
466
|
|
|
558
|
-
|
|
467
|
+
### ConsumerConfiguration
|
|
559
468
|
|
|
560
|
-
|
|
469
|
+
```ts
|
|
470
|
+
const consumer = client.createConsumer({
|
|
471
|
+
groupId: 'orders-worker',
|
|
472
|
+
enableAutoCommit: false,
|
|
473
|
+
fetchMetadataTimeout: 5000,
|
|
474
|
+
configuration: {
|
|
475
|
+
'auto.offset.reset': 'earliest',
|
|
476
|
+
'enable.auto.commit': false,
|
|
477
|
+
'fetch.min.bytes': 1,
|
|
478
|
+
},
|
|
479
|
+
})
|
|
480
|
+
```
|
|
561
481
|
|
|
562
|
-
|
|
|
563
|
-
|
|
|
564
|
-
| `
|
|
565
|
-
| `
|
|
566
|
-
| `
|
|
567
|
-
| `
|
|
568
|
-
| `brokerAddressFamily` | `string` | `"v4"` | Address family to use for the connection (v4, v6) |
|
|
569
|
-
| `configuration` | `Record<string, any>` | `{}` | Additional configuration options for the client |
|
|
570
|
-
| `diagnostics` | `boolean` | `false` | **v3.0.0+**: Enable diagnostics channel for OTEL instrumentation |
|
|
482
|
+
| Option | Type | Description |
|
|
483
|
+
| ---------------------- | --------------------- | ------------------------------------------ |
|
|
484
|
+
| `groupId` | `string` | Kafka consumer group id. |
|
|
485
|
+
| `enableAutoCommit` | `boolean` | Convenience flag for auto commit behavior. |
|
|
486
|
+
| `fetchMetadataTimeout` | `number` | Metadata fetch timeout in milliseconds. |
|
|
487
|
+
| `configuration` | `Record<string, any>` | Additional librdkafka consumer settings. |
|
|
571
488
|
|
|
572
|
-
###
|
|
489
|
+
### ProducerConfiguration
|
|
573
490
|
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
491
|
+
```ts
|
|
492
|
+
const producer = client.createProducer({
|
|
493
|
+
autoFlush: true,
|
|
494
|
+
queueTimeout: 5000,
|
|
495
|
+
configuration: {
|
|
496
|
+
'compression.type': 'lz4',
|
|
497
|
+
},
|
|
498
|
+
})
|
|
499
|
+
```
|
|
581
500
|
|
|
582
|
-
|
|
501
|
+
| Option | Type | Description |
|
|
502
|
+
| --------------- | --------------------- | ------------------------------------------------------- |
|
|
503
|
+
| `autoFlush` | `boolean` | When enabled, `send()` waits for delivery confirmation. |
|
|
504
|
+
| `queueTimeout` | `number` | Queue timeout in milliseconds. |
|
|
505
|
+
| `configuration` | `Record<string, any>` | Additional librdkafka producer settings. |
|
|
583
506
|
|
|
584
|
-
|
|
507
|
+
### TopicPartitionConfig
|
|
585
508
|
|
|
586
|
-
|
|
|
587
|
-
|
|
|
588
|
-
| `
|
|
589
|
-
| `
|
|
509
|
+
| Option | Type | Description |
|
|
510
|
+
| ----------------- | ------------------- | ------------------------------------------------ |
|
|
511
|
+
| `topic` | `string` | Topic name. |
|
|
512
|
+
| `allOffsets` | `OffsetModel` | Offset model applied to all assigned partitions. |
|
|
513
|
+
| `partitionOffset` | `PartitionOffset[]` | Partition-specific offset model. |
|
|
514
|
+
| `createTopic` | `boolean` | Create the topic before subscribing. |
|
|
515
|
+
| `numPartitions` | `number` | Partition count when creating a topic. |
|
|
516
|
+
| `replicas` | `number` | Replica count when creating a topic. |
|
|
590
517
|
|
|
591
|
-
|
|
592
|
-
// Using commitMessage (recommended for v2.1.0+)
|
|
593
|
-
const message = await consumer.recv()
|
|
594
|
-
await consumer.commitMessage(message, 'Sync')
|
|
518
|
+
Offset models:
|
|
595
519
|
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
520
|
+
```ts
|
|
521
|
+
{
|
|
522
|
+
position: 'Beginning'
|
|
523
|
+
}
|
|
524
|
+
{
|
|
525
|
+
position: 'End'
|
|
526
|
+
}
|
|
527
|
+
{
|
|
528
|
+
position: 'Stored'
|
|
529
|
+
}
|
|
530
|
+
{
|
|
531
|
+
position: 'Invalid'
|
|
532
|
+
}
|
|
533
|
+
{
|
|
534
|
+
offset: 42
|
|
535
|
+
}
|
|
599
536
|
```
|
|
600
537
|
|
|
601
|
-
|
|
538
|
+
### Type-Only Literals
|
|
602
539
|
|
|
603
|
-
|
|
540
|
+
In v4, these names are TypeScript-only exports:
|
|
604
541
|
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
| `autoFlush` | `boolean` | `true` | Enable automatic message flushing |
|
|
609
|
-
| `configuration` | `Record<string, any>` | `{}` | Additional producer configuration options |
|
|
610
|
-
|
|
611
|
-
### TopicPartitionConfig
|
|
542
|
+
```ts
|
|
543
|
+
import type { CommitMode, KafkaEventName, PartitionPosition, SecurityProtocol } from 'kafka-crab-js'
|
|
544
|
+
```
|
|
612
545
|
|
|
613
|
-
|
|
614
|
-
| ----------------- | ------------------------ | ------- | ----------------------------------------------------- |
|
|
615
|
-
| `topic` | `string` | | Topic name |
|
|
616
|
-
| `allOffsets` | `OffsetModel` | | Offset configuration for all partitions |
|
|
617
|
-
| `partitionOffset` | `Array<PartitionOffset>` | | Per-partition offset configuration |
|
|
618
|
-
| `createTopic` | `boolean` | `false` | **v2.0.0+**: Create topic if it doesn't exist |
|
|
619
|
-
| `numPartitions` | `number` | `1` | **v2.0.0+**: Number of partitions when creating topic |
|
|
620
|
-
| `replicas` | `number` | `1` | **v2.0.0+**: Number of replicas when creating topic |
|
|
546
|
+
Use literal values at runtime:
|
|
621
547
|
|
|
622
|
-
|
|
548
|
+
```ts
|
|
549
|
+
const commitMode: CommitMode = 'Sync'
|
|
550
|
+
const securityProtocol: SecurityProtocol = 'Plaintext'
|
|
551
|
+
```
|
|
623
552
|
|
|
624
|
-
|
|
553
|
+
## OpenTelemetry
|
|
625
554
|
|
|
626
|
-
|
|
555
|
+
OpenTelemetry support lives in the separate `kafka-crab-js-otel` package. The core package emits diagnostic-channel
|
|
556
|
+
events when `diagnostics !== false`; the OTEL package subscribes to those events.
|
|
627
557
|
|
|
628
|
-
|
|
558
|
+
Install the instrumentation package and OpenTelemetry dependencies:
|
|
629
559
|
|
|
630
560
|
```bash
|
|
631
|
-
npm install kafka-crab-js-otel @opentelemetry/api
|
|
561
|
+
npm install kafka-crab-js-otel @opentelemetry/api @opentelemetry/sdk-node
|
|
632
562
|
```
|
|
633
563
|
|
|
634
|
-
|
|
564
|
+
Enable instrumentation before creating `KafkaClient`:
|
|
635
565
|
|
|
636
|
-
```
|
|
566
|
+
```ts
|
|
637
567
|
import { KafkaClient } from 'kafka-crab-js'
|
|
638
568
|
import { enableOtelInstrumentation, endSpan } from 'kafka-crab-js-otel'
|
|
639
569
|
|
|
640
|
-
// Enable instrumentation before creating client
|
|
641
|
-
// Note: serviceName is configured via OTEL SDK Resource
|
|
642
570
|
enableOtelInstrumentation({
|
|
643
571
|
metrics: { enabled: true },
|
|
644
572
|
})
|
|
645
573
|
|
|
646
|
-
// Create client with diagnostics enabled
|
|
647
574
|
const client = new KafkaClient({
|
|
648
575
|
brokers: 'localhost:9092',
|
|
649
|
-
clientId: '
|
|
576
|
+
clientId: 'orders-worker',
|
|
650
577
|
diagnostics: true,
|
|
651
578
|
})
|
|
652
579
|
|
|
653
|
-
|
|
580
|
+
const consumer = client.createConsumer({ groupId: 'orders-worker' })
|
|
581
|
+
await consumer.subscribe('orders')
|
|
582
|
+
|
|
654
583
|
const message = await consumer.recv()
|
|
655
|
-
|
|
656
|
-
|
|
584
|
+
if (message) {
|
|
585
|
+
try {
|
|
586
|
+
await processOrder(message.payload)
|
|
587
|
+
} finally {
|
|
588
|
+
endSpan(message)
|
|
589
|
+
}
|
|
590
|
+
}
|
|
657
591
|
```
|
|
658
592
|
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
#### Examples
|
|
662
|
-
|
|
663
|
-
See comprehensive examples in the `example/` directory:
|
|
664
|
-
|
|
665
|
-
- `example/otel-tracing-example.mjs` - Complete tracing setup with Jaeger
|
|
666
|
-
- `example/otel-metrics-example.mjs` - Metrics collection with Prometheus
|
|
667
|
-
- `example/README.md` - Full documentation for all examples
|
|
593
|
+
See the [OpenTelemetry package README](../otel/README.md) for full tracing and metrics setup.
|
|
668
594
|
|
|
669
595
|
## Performance Benchmarks
|
|
670
596
|
|
|
671
|
-
|
|
597
|
+
The repository includes a benchmark suite that compares `kafka-crab-js` with KafkaJS and `@platformatic/kafka`.
|
|
672
598
|
|
|
673
|
-
|
|
599
|
+
From the repository root:
|
|
674
600
|
|
|
675
601
|
```bash
|
|
676
|
-
cd
|
|
602
|
+
cd packages/benchmark
|
|
677
603
|
|
|
678
|
-
# Set up benchmark
|
|
604
|
+
# Set up benchmark data. Requires Kafka running locally.
|
|
679
605
|
vp install
|
|
680
606
|
vp run setup:consumer
|
|
681
607
|
|
|
682
|
-
# Run
|
|
608
|
+
# Run the default isolated-process memory benchmark.
|
|
683
609
|
vp run benchmark
|
|
684
610
|
```
|
|
685
611
|
|
|
686
|
-
|
|
612
|
+
The default benchmark runs one child Node.js process per selected scenario. This avoids carrying V8 heap pages, native
|
|
613
|
+
allocator arenas, sockets, librdkafka state, and Kafka metadata from one client into the next client's measurement.
|
|
687
614
|
|
|
688
|
-
|
|
615
|
+
### Consumer Benchmark Snapshot
|
|
689
616
|
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
║ node-rdkafka (evented) │ 84115 │ 24922.67 op/sec │ ± 74.82 % │ + 2887.91 % ║
|
|
696
|
-
║ kafka-crab-js (serial) │ 50000 │ 43213.86 op/sec │ ± 3.46 % │ + 73.39 % ║
|
|
697
|
-
║ node-rdkafka (stream) │ 50000 │ 49805.32 op/sec │ ± 27.10 % │ + 15.25 % ║
|
|
698
|
-
╟────────────────────────┼─────────┼──────────────────┼───────────┼──────────────────────────╢
|
|
699
|
-
║ Fastest test │ Samples │ Result │ Tolerance │ Difference with previous ║
|
|
700
|
-
╟────────────────────────┼─────────┼──────────────────┼───────────┼──────────────────────────╢
|
|
701
|
-
║ kafka-crab-js (batch) │ 50000 │ 205985.31 op/sec │ ± 16.53 % │ + 313.58 % ║
|
|
702
|
-
╚════════════════════════╧═════════╧══════════════════╧═══════════╧══════════════════════════╝
|
|
617
|
+
The captured run below is equivalent to:
|
|
618
|
+
|
|
619
|
+
```bash
|
|
620
|
+
cd packages/benchmark
|
|
621
|
+
BENCHMARK_BATCH_SIZE=300000 BENCHMARK_BATCH_TIMEOUT_MS=2 vp run benchmark
|
|
703
622
|
```
|
|
704
623
|
|
|
705
|
-
|
|
624
|
+
Batch scenarios were normalized to the comparable effective batch size of `16384`.
|
|
706
625
|
|
|
707
|
-
|
|
708
|
-
- **kafka-crab-js (batch)**: Batch message processing - **205,985 ops/sec** (fastest)
|
|
709
|
-
- **node-rdkafka (evented)**: Event-based processing - **24,923 ops/sec**
|
|
710
|
-
- **node-rdkafka (stream)**: Stream-based processing - **49,805 ops/sec**
|
|
711
|
-
- **kafkajs**: Official KafkaJS client - **834 ops/sec**
|
|
626
|
+

|
|
712
627
|
|
|
713
|
-
|
|
628
|
+
#### Throughput
|
|
714
629
|
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
630
|
+
| Rank | Scenario | Result | Relative |
|
|
631
|
+
| ---: | ----------------------------------- | --------------------: | -------: |
|
|
632
|
+
| 1 | `kafka-crab-js v4 (stream, batch)` | `1,089,284.97 op/sec` | `100.0%` |
|
|
633
|
+
| 2 | `@platformatic/kafka` | `732,207.27 op/sec` | `67.2%` |
|
|
634
|
+
| 3 | `KafkaJS (eachBatch)` | `676,576.98 op/sec` | `62.1%` |
|
|
635
|
+
| 4 | `kafka-crab-js v4 (stream, serial)` | `544,658.33 op/sec` | `50.0%` |
|
|
636
|
+
| 5 | `KafkaJS (eachMessage, concurrent)` | `509,082.43 op/sec` | `46.7%` |
|
|
637
|
+
| 6 | `KafkaJS (eachMessage)` | `505,815.57 op/sec` | `46.4%` |
|
|
720
638
|
|
|
721
|
-
|
|
639
|
+
`kafka-crab-js v4 (stream, batch)` is the fastest scenario in this run. It is about `49%` faster than
|
|
640
|
+
`@platformatic/kafka` and about `61%` faster than `KafkaJS (eachBatch)`. Its tolerance is higher than the other top
|
|
641
|
+
rows (`+/- 9.15%`), so the exact gap should be treated as a benchmark snapshot rather than a universal constant.
|
|
722
642
|
|
|
723
|
-
|
|
643
|
+
For message-by-message consumption, `kafka-crab-js v4 (stream, serial)` is close to KafkaJS throughput while using much
|
|
644
|
+
less lifecycle memory. In this run it is about `7.7%` faster than `KafkaJS (eachMessage)` and about `7.0%` faster than
|
|
645
|
+
the concurrent KafkaJS `eachMessage` scenario.
|
|
724
646
|
|
|
725
|
-
|
|
726
|
-
export const topic = 'benchmarks'
|
|
727
|
-
export const brokers = ['localhost:9092', 'localhost:9093', 'localhost:9094']
|
|
728
|
-
|
|
729
|
-
// Benchmark parameters can be adjusted in consumer.ts:
|
|
730
|
-
const iterations = 10_000 // Number of messages to process
|
|
731
|
-
const maxBytes = 200 // Maximum message size
|
|
732
|
-
```
|
|
647
|
+
#### Memory And GC
|
|
733
648
|
|
|
734
|
-
|
|
649
|
+
| Scenario | RSS delta | Peak heap | GC time | GC share | Notes |
|
|
650
|
+
| ----------------------------------- | ----------: | ---------: | ---------: | -------: | ----------------------------------------------- |
|
|
651
|
+
| `kafka-crab-js v4 (stream, serial)` | `72.3 MiB` | `11.7 MiB` | `36.75 ms` | `4.00%` | Best memory efficiency. |
|
|
652
|
+
| `kafka-crab-js v4 (stream, batch)` | `167.0 MiB` | `35.9 MiB` | `30.60 ms` | `6.67%` | Best throughput and lowest GC time. |
|
|
653
|
+
| `KafkaJS (eachBatch)` | `193.1 MiB` | `70.0 MiB` | `63.03 ms` | `8.53%` | Strong batch baseline. |
|
|
654
|
+
| `@platformatic/kafka` | `217.6 MiB` | `93.7 MiB` | `73.62 ms` | `10.78%` | Fastest non-crab competitor. |
|
|
655
|
+
| `KafkaJS (eachMessage)` | `181.5 MiB` | `77.0 MiB` | `79.11 ms` | `8.00%` | Similar throughput to v4 serial, higher memory. |
|
|
656
|
+
| `KafkaJS (eachMessage, concurrent)` | `188.2 MiB` | `75.6 MiB` | `77.17 ms` | `7.86%` | Concurrency did not improve this workload. |
|
|
735
657
|
|
|
736
|
-
|
|
658
|
+
Lifecycle memory includes module loading, client creation, subscription, consumption, cleanup, and retained RSS after
|
|
659
|
+
cleanup. GC metrics use the narrower first-to-last-message window used for throughput.
|
|
737
660
|
|
|
738
|
-
|
|
739
|
-
-
|
|
740
|
-
|
|
661
|
+
The most important memory result is efficiency, not just raw RSS. `kafka-crab-js v4 (stream, serial)` delivered
|
|
662
|
+
`7529 op/sec/MiB`, the best score in the run. `kafka-crab-js v4 (stream, batch)` delivered `6521 op/sec/MiB`, ahead of
|
|
663
|
+
`KafkaJS (eachBatch)` at `3504 op/sec/MiB` and `@platformatic/kafka` at `3365 op/sec/MiB`.
|
|
741
664
|
|
|
742
|
-
###
|
|
665
|
+
### Benchmark Interpretation
|
|
743
666
|
|
|
744
|
-
- Use batch
|
|
745
|
-
-
|
|
746
|
-
-
|
|
747
|
-
-
|
|
667
|
+
- Use `kafka-crab-js v4 (stream, batch)` when raw throughput is the priority.
|
|
668
|
+
- Use `kafka-crab-js v4 (stream, serial)` when memory efficiency and low heap pressure matter most.
|
|
669
|
+
- Compare batch scenarios with batch scenarios and message scenarios with message scenarios.
|
|
670
|
+
- `@platformatic/kafka` is the strongest non-crab throughput competitor in this run, but it used more RSS, heap, and GC
|
|
671
|
+
time than the v4 batch scenario.
|
|
672
|
+
- KafkaJS `eachMessage` concurrency did not help this workload.
|
|
748
673
|
|
|
749
|
-
### Configuration
|
|
750
|
-
|
|
751
|
-
- Use the flexible configuration system with proper data types:
|
|
752
|
-
```javascript
|
|
753
|
-
const config = {
|
|
754
|
-
'batch.size': 16384, // number
|
|
755
|
-
'compression.type': 'snappy', // string
|
|
756
|
-
'enable.idempotence': true, // boolean
|
|
757
|
-
retries: 5, // number
|
|
758
|
-
}
|
|
759
|
-
```
|
|
674
|
+
### Benchmark Configuration
|
|
760
675
|
|
|
761
|
-
|
|
676
|
+
Common knobs:
|
|
762
677
|
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
678
|
+
```bash
|
|
679
|
+
BENCHMARK_ITERATIONS=100000
|
|
680
|
+
BENCHMARK_RUNS=5
|
|
681
|
+
BENCHMARK_BATCH_SIZE=4096
|
|
682
|
+
BENCHMARK_BATCH_TIMEOUT_MS=2
|
|
683
|
+
BENCHMARK_MAX_BYTES=2048
|
|
684
|
+
BENCHMARK_MEMORY=1
|
|
685
|
+
```
|
|
767
686
|
|
|
768
|
-
|
|
687
|
+
See the [benchmark README](../benchmark/README.md) for the full benchmark methodology and environment variables.
|
|
769
688
|
|
|
770
|
-
|
|
771
|
-
- Use `objectMode: true` for structured message processing
|
|
772
|
-
- Set appropriate `highWaterMark` based on memory constraints
|
|
689
|
+
## Migration Notes
|
|
773
690
|
|
|
774
|
-
|
|
691
|
+
### v4 Type-Only Runtime Exports
|
|
775
692
|
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
3. Run the benchmark suite to ensure performance isn't degraded
|
|
779
|
-
4. Commit your changes (`git commit -m 'Add some amazing feature'`)
|
|
780
|
-
5. Push to the branch (`git push origin feature/amazing-feature`)
|
|
781
|
-
6. Open a Pull Request
|
|
693
|
+
`CommitMode`, `KafkaEventName`, `PartitionPosition`, and `SecurityProtocol` are no longer runtime exports from
|
|
694
|
+
`kafka-crab-js`.
|
|
782
695
|
|
|
783
|
-
|
|
696
|
+
Before:
|
|
784
697
|
|
|
785
|
-
```
|
|
786
|
-
|
|
787
|
-
|
|
698
|
+
```ts
|
|
699
|
+
import { CommitMode, SecurityProtocol } from 'kafka-crab-js'
|
|
700
|
+
```
|
|
788
701
|
|
|
789
|
-
|
|
790
|
-
vp run typecheck
|
|
702
|
+
After:
|
|
791
703
|
|
|
792
|
-
|
|
793
|
-
|
|
704
|
+
```ts
|
|
705
|
+
import type { CommitMode, SecurityProtocol } from 'kafka-crab-js'
|
|
794
706
|
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
vp run benchmark
|
|
707
|
+
const commitMode: CommitMode = 'Sync'
|
|
708
|
+
const securityProtocol: SecurityProtocol = 'Plaintext'
|
|
798
709
|
```
|
|
799
710
|
|
|
800
|
-
|
|
711
|
+
### v3 OpenTelemetry Split
|
|
801
712
|
|
|
802
|
-
|
|
713
|
+
OpenTelemetry instrumentation moved from the core package to `kafka-crab-js-otel` in v3. This keeps the core package
|
|
714
|
+
small and makes tracing/metrics opt-in.
|
|
803
715
|
|
|
804
|
-
|
|
716
|
+
## Troubleshooting
|
|
805
717
|
|
|
806
|
-
|
|
807
|
-
- **Consumer & stream coverage** – Standard consumers, batch consumers, and `createStreamConsumer` streams emit spans that include consumer group, topic, partition, offset, and batch size semantics.
|
|
808
|
-
- **Hook-friendly spans** – Both `messageHook` and `producerHook` callbacks run inside the active span context, simplifying attribute decoration or error handling.
|
|
809
|
-
- **Zero overhead when disabled** – Uses Node.js `diagnostics_channel` for near-zero cost when OTEL is not active.
|
|
718
|
+
### The Consumer Does Not Receive Existing Messages
|
|
810
719
|
|
|
811
|
-
|
|
720
|
+
Use an offset reset policy and a fresh consumer group:
|
|
812
721
|
|
|
813
|
-
|
|
722
|
+
```ts
|
|
723
|
+
const consumer = client.createConsumer({
|
|
724
|
+
groupId: `orders-worker-${Date.now()}`,
|
|
725
|
+
configuration: {
|
|
726
|
+
'auto.offset.reset': 'earliest',
|
|
727
|
+
},
|
|
728
|
+
})
|
|
729
|
+
```
|
|
814
730
|
|
|
815
|
-
|
|
816
|
-
- For batch consumers, call `endSpan(batch)` when you're done processing the batch.
|
|
817
|
-
- The `endSpan()` helper is exported from `kafka-crab-js-otel`.
|
|
731
|
+
Or subscribe with an explicit starting position:
|
|
818
732
|
|
|
819
|
-
|
|
733
|
+
```ts
|
|
734
|
+
await consumer.subscribe([{ topic: 'orders', allOffsets: { position: 'Beginning' } }])
|
|
735
|
+
```
|
|
820
736
|
|
|
821
|
-
###
|
|
737
|
+
### The Process Hangs On Shutdown
|
|
822
738
|
|
|
823
|
-
|
|
739
|
+
Direct consumers should call `disconnect()`:
|
|
824
740
|
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
741
|
+
```ts
|
|
742
|
+
await consumer.disconnect()
|
|
743
|
+
```
|
|
828
744
|
|
|
829
|
-
|
|
745
|
+
Node stream consumers should call `destroy()`:
|
|
746
|
+
|
|
747
|
+
```ts
|
|
748
|
+
stream.destroy()
|
|
749
|
+
```
|
|
830
750
|
|
|
831
|
-
|
|
751
|
+
Web stream consumers should cancel the reader and disconnect the underlying consumer:
|
|
832
752
|
|
|
833
753
|
```ts
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
import { SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'
|
|
838
|
-
import { AsyncHooksContextManager } from '@opentelemetry/context-async-hooks'
|
|
839
|
-
import { context } from '@opentelemetry/api'
|
|
840
|
-
|
|
841
|
-
// 1. Set up OpenTelemetry SDK
|
|
842
|
-
const provider = new NodeTracerProvider()
|
|
843
|
-
provider.addSpanProcessor(new SimpleSpanProcessor(exporter))
|
|
844
|
-
provider.register()
|
|
845
|
-
context.setGlobalContextManager(new AsyncHooksContextManager().enable())
|
|
846
|
-
|
|
847
|
-
// 2. Enable kafka-crab-js instrumentation BEFORE creating client
|
|
848
|
-
enableOtelInstrumentation({
|
|
849
|
-
captureMessagePayload: true,
|
|
850
|
-
captureMessageHeaders: true,
|
|
851
|
-
producerHook: (span) => span.setAttribute('messaging.client.kind', 'producer'),
|
|
852
|
-
})
|
|
754
|
+
await reader.cancel()
|
|
755
|
+
await webConsumer.consumer.disconnect()
|
|
756
|
+
```
|
|
853
757
|
|
|
854
|
-
|
|
855
|
-
const client = new KafkaClient({
|
|
856
|
-
brokers: 'localhost:29092',
|
|
857
|
-
clientId: 'orders-api',
|
|
858
|
-
diagnostics: true, // Required for OTEL to receive events
|
|
859
|
-
})
|
|
758
|
+
### Localhost Resolves To IPv6 But Kafka Listens On IPv4
|
|
860
759
|
|
|
861
|
-
|
|
862
|
-
const producer = client.createProducer()
|
|
863
|
-
await producer.send({
|
|
864
|
-
topic: 'orders',
|
|
865
|
-
messages: [
|
|
866
|
-
{
|
|
867
|
-
payload: Buffer.from(JSON.stringify({ orderId: '123' })),
|
|
868
|
-
headers: { 'custom-header': 'foo' },
|
|
869
|
-
},
|
|
870
|
-
],
|
|
871
|
-
})
|
|
760
|
+
Set `brokerAddressFamily: 'v4'`:
|
|
872
761
|
|
|
873
|
-
|
|
874
|
-
const
|
|
875
|
-
|
|
876
|
-
|
|
762
|
+
```ts
|
|
763
|
+
const client = new KafkaClient({
|
|
764
|
+
brokers: 'localhost:9092',
|
|
765
|
+
brokerAddressFamily: 'v4',
|
|
877
766
|
})
|
|
767
|
+
```
|
|
878
768
|
|
|
879
|
-
|
|
769
|
+
### Need Advanced Kafka Settings
|
|
880
770
|
|
|
881
|
-
|
|
882
|
-
try {
|
|
883
|
-
console.log(message.headers?.['custom-header']?.toString())
|
|
884
|
-
} finally {
|
|
885
|
-
// IMPORTANT: close the "process <topic>" span when your app finishes handling the message
|
|
886
|
-
endSpan(message)
|
|
887
|
-
}
|
|
888
|
-
})
|
|
771
|
+
Pass librdkafka settings through `configuration`:
|
|
889
772
|
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
773
|
+
```ts
|
|
774
|
+
const client = new KafkaClient({
|
|
775
|
+
brokers: 'localhost:9092',
|
|
776
|
+
configuration: {
|
|
777
|
+
'socket.keepalive.enable': true,
|
|
778
|
+
'metadata.max.age.ms': 300000,
|
|
779
|
+
},
|
|
780
|
+
})
|
|
893
781
|
```
|
|
894
782
|
|
|
895
|
-
|
|
783
|
+
Consumer and producer configuration objects also accept their own `configuration` maps.
|
|
896
784
|
|
|
897
|
-
##
|
|
785
|
+
## Development
|
|
898
786
|
|
|
899
|
-
|
|
787
|
+
From the repository root:
|
|
900
788
|
|
|
901
|
-
|
|
789
|
+
```bash
|
|
790
|
+
vp install
|
|
791
|
+
vp run build
|
|
792
|
+
vp run test
|
|
793
|
+
vp run test:integration
|
|
794
|
+
vp check
|
|
795
|
+
```
|
|
902
796
|
|
|
903
|
-
|
|
797
|
+
Useful package-local commands:
|
|
904
798
|
|
|
905
|
-
|
|
799
|
+
```bash
|
|
800
|
+
vp run --filter kafka-crab-js build
|
|
801
|
+
vp run --filter kafka-crab-js test
|
|
802
|
+
vp run --filter kafka-crab-js test:integration
|
|
803
|
+
```
|
|
804
|
+
|
|
805
|
+
Integration tests require Kafka at `localhost:9092` unless `KAFKA_BROKERS` is set.
|
|
806
|
+
|
|
807
|
+
## License
|
|
906
808
|
|
|
907
|
-
|
|
809
|
+
MIT
|