kafka-crab-js 3.1.0 → 4.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +239 -170
- package/dist/base-kafka-stream-readable-BokUK06Z.d.ts +69 -0
- package/dist/base-kafka-stream-readable-I1xda63L.d.cts +69 -0
- package/dist/channels-CYQ4ni8K.d.cts +133 -0
- package/dist/channels-QBobKRC2.d.ts +133 -0
- package/dist/diagnostics/channels.cjs +85 -0
- package/dist/diagnostics/channels.cjs.map +1 -0
- package/dist/diagnostics/channels.d.cts +2 -0
- package/dist/diagnostics/channels.d.ts +2 -138
- package/dist/diagnostics/channels.js +72 -0
- package/dist/diagnostics/channels.js.map +1 -0
- package/dist/diagnostics/index.cjs +21 -0
- package/dist/diagnostics/index.d.cts +3 -0
- package/dist/diagnostics/index.d.ts +3 -7
- package/dist/diagnostics/index.js +3 -0
- package/dist/diagnostics/instrumentation.cjs +334 -0
- package/dist/diagnostics/instrumentation.cjs.map +1 -0
- package/dist/diagnostics/instrumentation.d.cts +2 -0
- package/dist/diagnostics/instrumentation.d.ts +2 -31
- package/dist/diagnostics/instrumentation.js +329 -0
- package/dist/diagnostics/instrumentation.js.map +1 -0
- package/dist/index.cjs +47 -701
- package/dist/index.d.cts +8 -0
- package/dist/index.d.ts +8 -8
- package/dist/index.js +9 -662
- package/dist/instrumentation-BiIPAe1B.d.cts +37 -0
- package/dist/instrumentation-CwoJPHwZ.d.ts +37 -0
- package/dist/kafka-client.cjs +314 -0
- package/dist/kafka-client.cjs.map +1 -0
- package/dist/kafka-client.d.cts +107 -0
- package/dist/kafka-client.d.ts +98 -52
- package/dist/kafka-client.js +313 -0
- package/dist/kafka-client.js.map +1 -0
- package/dist/streams/base-kafka-stream-readable.cjs +109 -0
- package/dist/streams/base-kafka-stream-readable.cjs.map +1 -0
- package/dist/streams/base-kafka-stream-readable.d.cts +2 -0
- package/dist/streams/base-kafka-stream-readable.d.ts +2 -60
- package/dist/streams/base-kafka-stream-readable.js +108 -0
- package/dist/streams/base-kafka-stream-readable.js.map +1 -0
- package/dist/streams/kafka-batch-stream-readable.cjs +106 -0
- package/dist/streams/kafka-batch-stream-readable.cjs.map +1 -0
- package/dist/streams/kafka-batch-stream-readable.d.cts +46 -0
- package/dist/streams/kafka-batch-stream-readable.d.ts +40 -25
- package/dist/streams/kafka-batch-stream-readable.js +105 -0
- package/dist/streams/kafka-batch-stream-readable.js.map +1 -0
- package/dist/streams/kafka-stream-readable.cjs +63 -0
- package/dist/streams/kafka-stream-readable.cjs.map +1 -0
- package/dist/streams/kafka-stream-readable.d.cts +27 -0
- package/dist/streams/kafka-stream-readable.d.ts +22 -7
- package/dist/streams/kafka-stream-readable.js +62 -0
- package/dist/streams/kafka-stream-readable.js.map +1 -0
- package/js-binding.cjs +52 -52
- package/js-binding.d.ts +32 -0
- package/js-binding.js +52 -52
- package/package.json +53 -52
- package/dist/index.cjs.map +0 -7
- package/dist/index.js.map +0 -7
package/README.md
CHANGED
|
@@ -6,10 +6,51 @@ A lightweight, flexible, and reliable Kafka client for JavaScript/TypeScript. It
|
|
|
6
6
|
|
|
7
7
|
[](https://www.npmjs.com/package/kafka-crab-js)
|
|
8
8
|
[](https://opensource.org/licenses/MIT)
|
|
9
|
+
|
|
9
10
|
</div>
|
|
10
11
|
|
|
11
12
|
---
|
|
12
13
|
|
|
14
|
+
## What's New in Version 4.0.0
|
|
15
|
+
|
|
16
|
+
### BREAKING CHANGES ⚠️
|
|
17
|
+
|
|
18
|
+
**`CommitMode`, `KafkaEventName`, `PartitionPosition`, and `SecurityProtocol` are no longer runtime exports from `kafka-crab-js`.**
|
|
19
|
+
|
|
20
|
+
These names are now type-only at the root package API. This change applies to the `kafka-crab-js` root import surface and is intended as a major-version change.
|
|
21
|
+
|
|
22
|
+
#### Migration from v3.x
|
|
23
|
+
|
|
24
|
+
**Before (v3.x):**
|
|
25
|
+
|
|
26
|
+
```ts
|
|
27
|
+
import { CommitMode, SecurityProtocol } from 'kafka-crab-js'
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
**After for type-only usage:**
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
import type { CommitMode, SecurityProtocol } from 'kafka-crab-js'
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
**After for runtime usage:**
|
|
37
|
+
|
|
38
|
+
```ts
|
|
39
|
+
const commitMode: 'Sync' | 'Async' = 'Sync'
|
|
40
|
+
const securityProtocol: 'Plaintext' | 'Ssl' | 'SaslPlaintext' | 'SaslSsl' = 'Plaintext'
|
|
41
|
+
const eventName: 'PreRebalance' | 'PostRebalance' | 'CommitCallback' = 'PreRebalance'
|
|
42
|
+
const partitionPosition: 'Beginning' | 'End' | 'Stored' | 'Invalid' = 'Beginning'
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
#### Replacement Values
|
|
46
|
+
|
|
47
|
+
| Name | Replacement string literal values |
|
|
48
|
+
| ------------------- | ------------------------------------------------------- |
|
|
49
|
+
| `CommitMode` | `'Sync' \| 'Async'` |
|
|
50
|
+
| `KafkaEventName` | `'PreRebalance' \| 'PostRebalance' \| 'CommitCallback'` |
|
|
51
|
+
| `PartitionPosition` | `'Beginning' \| 'End' \| 'Stored' \| 'Invalid'` |
|
|
52
|
+
| `SecurityProtocol` | `'Plaintext' \| 'Ssl' \| 'SaslPlaintext' \| 'SaslSsl'` |
|
|
53
|
+
|
|
13
54
|
## What's New in Version 3.0.0
|
|
14
55
|
|
|
15
56
|
### BREAKING CHANGES ⚠️
|
|
@@ -21,6 +62,7 @@ This change reduces the core package size and makes OTEL an opt-in dependency.
|
|
|
21
62
|
#### Migration from v2.x
|
|
22
63
|
|
|
23
64
|
**Before (v2.x):**
|
|
65
|
+
|
|
24
66
|
```javascript
|
|
25
67
|
import { KafkaClient } from 'kafka-crab-js'
|
|
26
68
|
|
|
@@ -35,6 +77,7 @@ const client = new KafkaClient({
|
|
|
35
77
|
```
|
|
36
78
|
|
|
37
79
|
**After (v3.x):**
|
|
80
|
+
|
|
38
81
|
```javascript
|
|
39
82
|
import { KafkaClient } from 'kafka-crab-js'
|
|
40
83
|
import { enableOtelInstrumentation, endSpan } from 'kafka-crab-js-otel'
|
|
@@ -60,21 +103,22 @@ endSpan(message)
|
|
|
60
103
|
|
|
61
104
|
#### Key Changes
|
|
62
105
|
|
|
63
|
-
| v2.x
|
|
64
|
-
|
|
106
|
+
| v2.x | v3.x |
|
|
107
|
+
| ---------------------------- | ------------------------------------------------------- |
|
|
65
108
|
| `otel` config in KafkaClient | `enableOtelInstrumentation()` from `kafka-crab-js-otel` |
|
|
66
|
-
| Automatic span ending
|
|
67
|
-
| N/A
|
|
68
|
-
| OTEL bundled
|
|
109
|
+
| Automatic span ending | Call `endSpan(message)` manually for consumers |
|
|
110
|
+
| N/A | `diagnostics: true` required in KafkaClient config |
|
|
111
|
+
| OTEL bundled | Install `kafka-crab-js-otel` separately |
|
|
69
112
|
|
|
70
113
|
#### New Package Structure
|
|
71
114
|
|
|
72
|
-
| Package
|
|
73
|
-
|
|
74
|
-
| `kafka-crab-js`
|
|
115
|
+
| Package | Description |
|
|
116
|
+
| -------------------- | ------------------------------------------------ |
|
|
117
|
+
| `kafka-crab-js` | Core Kafka client (producer, consumer, streams) |
|
|
75
118
|
| `kafka-crab-js-otel` | OpenTelemetry instrumentation (separate install) |
|
|
76
119
|
|
|
77
120
|
### Internal Improvements
|
|
121
|
+
|
|
78
122
|
- Uses Node.js `diagnostics_channel` for observability (zero overhead when not subscribed)
|
|
79
123
|
- Cleaner separation of concerns between core and observability
|
|
80
124
|
- Smaller bundle size for users who don't need OTEL
|
|
@@ -94,13 +138,13 @@ endSpan(message)
|
|
|
94
138
|
- Available on both `KafkaConsumer` and stream consumers
|
|
95
139
|
- **Before** (v2.0.0):
|
|
96
140
|
```javascript
|
|
97
|
-
const message = await consumer.recv()
|
|
98
|
-
await consumer.commit(message.topic, message.partition, message.offset + 1, 'Sync')
|
|
141
|
+
const message = await consumer.recv()
|
|
142
|
+
await consumer.commit(message.topic, message.partition, message.offset + 1, 'Sync')
|
|
99
143
|
```
|
|
100
144
|
- **After** (v2.1.0):
|
|
101
145
|
```javascript
|
|
102
|
-
const message = await consumer.recv()
|
|
103
|
-
await consumer.commitMessage(message, 'Sync')
|
|
146
|
+
const message = await consumer.recv()
|
|
147
|
+
await consumer.commitMessage(message, 'Sync')
|
|
104
148
|
```
|
|
105
149
|
|
|
106
150
|
2. **Enhanced OpenTelemetry Support**:
|
|
@@ -179,14 +223,14 @@ pnpm add kafka-crab-js
|
|
|
179
223
|
### Basic Consumer Setup
|
|
180
224
|
|
|
181
225
|
```javascript
|
|
182
|
-
import { KafkaClient } from 'kafka-crab-js'
|
|
226
|
+
import { KafkaClient } from 'kafka-crab-js'
|
|
183
227
|
async function run() {
|
|
184
228
|
const kafkaClient = new KafkaClient({
|
|
185
229
|
brokers: 'localhost:29092',
|
|
186
230
|
clientId: 'foo-client',
|
|
187
231
|
logLevel: 'debug',
|
|
188
232
|
brokerAddressFamily: 'v4',
|
|
189
|
-
})
|
|
233
|
+
})
|
|
190
234
|
|
|
191
235
|
// Create consumer with topic creation control
|
|
192
236
|
const consumer = kafkaClient.createConsumer({
|
|
@@ -194,77 +238,81 @@ async function run() {
|
|
|
194
238
|
configuration: {
|
|
195
239
|
'auto.offset.reset': 'earliest',
|
|
196
240
|
'enable.auto.commit': false, // Use manual commit for better control
|
|
197
|
-
}
|
|
198
|
-
})
|
|
241
|
+
},
|
|
242
|
+
})
|
|
199
243
|
|
|
200
244
|
// Subscribe with topic creation options
|
|
201
|
-
await consumer.subscribe([
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
245
|
+
await consumer.subscribe([
|
|
246
|
+
{
|
|
247
|
+
topic: 'foo',
|
|
248
|
+
createTopic: true,
|
|
249
|
+
numPartitions: 3,
|
|
250
|
+
},
|
|
251
|
+
])
|
|
252
|
+
|
|
253
|
+
const message = await consumer.recv()
|
|
254
|
+
const { payload, partition, offset, topic } = message
|
|
209
255
|
console.log({
|
|
210
256
|
topic,
|
|
211
257
|
partition,
|
|
212
258
|
offset,
|
|
213
|
-
value: payload.toString()
|
|
214
|
-
})
|
|
259
|
+
value: payload.toString(),
|
|
260
|
+
})
|
|
215
261
|
|
|
216
262
|
// Manual commit - two options:
|
|
217
263
|
// Option 1 (v2.1.0+): Simplified with commitMessage
|
|
218
|
-
await consumer.commitMessage(message, 'Sync')
|
|
219
|
-
|
|
264
|
+
await consumer.commitMessage(message, 'Sync')
|
|
265
|
+
|
|
220
266
|
// Option 2: Traditional commit with manual offset increment
|
|
221
267
|
// await consumer.commit(topic, partition, offset + 1, 'Sync');
|
|
222
268
|
|
|
223
|
-
consumer.unsubscribe()
|
|
269
|
+
consumer.unsubscribe()
|
|
224
270
|
}
|
|
225
271
|
|
|
226
|
-
await run()
|
|
272
|
+
await run()
|
|
227
273
|
```
|
|
228
274
|
|
|
229
275
|
### Basic Producer Setup
|
|
230
276
|
|
|
231
277
|
```javascript
|
|
232
|
-
import { KafkaClient } from 'kafka-crab-js'
|
|
278
|
+
import { KafkaClient } from 'kafka-crab-js'
|
|
233
279
|
|
|
234
280
|
const kafkaClient = new KafkaClient({
|
|
235
281
|
brokers: 'localhost:29092',
|
|
236
282
|
clientId: 'my-client-id',
|
|
237
283
|
logLevel: 'info',
|
|
238
284
|
brokerAddressFamily: 'v4',
|
|
239
|
-
})
|
|
285
|
+
})
|
|
240
286
|
|
|
241
287
|
// Producer configuration is now optional with sensible defaults
|
|
242
288
|
const producer = kafkaClient.createProducer({
|
|
243
289
|
configuration: {
|
|
244
|
-
'message.timeout.ms': 5000,
|
|
290
|
+
'message.timeout.ms': 5000, // Now supports number values
|
|
245
291
|
'batch.size': 16384,
|
|
246
|
-
'compression.type': 'snappy'
|
|
247
|
-
}
|
|
248
|
-
})
|
|
292
|
+
'compression.type': 'snappy',
|
|
293
|
+
},
|
|
294
|
+
})
|
|
249
295
|
|
|
250
296
|
const message = {
|
|
251
297
|
id: 1,
|
|
252
|
-
name:
|
|
253
|
-
timestamp: new Date().toISOString()
|
|
254
|
-
}
|
|
298
|
+
name: 'Sample Message',
|
|
299
|
+
timestamp: new Date().toISOString(),
|
|
300
|
+
}
|
|
255
301
|
|
|
256
302
|
const result = await producer.send({
|
|
257
303
|
topic: 'my-topic',
|
|
258
|
-
messages: [
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
}
|
|
304
|
+
messages: [
|
|
305
|
+
{
|
|
306
|
+
payload: Buffer.from(JSON.stringify(message)),
|
|
307
|
+
},
|
|
308
|
+
],
|
|
309
|
+
})
|
|
262
310
|
|
|
263
|
-
const errors = result.map(r => r.error).filter(Boolean)
|
|
311
|
+
const errors = result.map((r) => r.error).filter(Boolean)
|
|
264
312
|
if (errors.length > 0) {
|
|
265
|
-
console.error('Error sending message:', errors)
|
|
313
|
+
console.error('Error sending message:', errors)
|
|
266
314
|
} else {
|
|
267
|
-
console.log('Message sent. Offset:', result)
|
|
315
|
+
console.log('Message sent. Offset:', result)
|
|
268
316
|
}
|
|
269
317
|
```
|
|
270
318
|
|
|
@@ -273,47 +321,50 @@ if (errors.length > 0) {
|
|
|
273
321
|
### Enhanced Stream Consumer Example
|
|
274
322
|
|
|
275
323
|
```javascript
|
|
276
|
-
import { KafkaClient } from 'kafka-crab-js'
|
|
324
|
+
import { KafkaClient } from 'kafka-crab-js'
|
|
277
325
|
|
|
278
326
|
const kafkaClient = new KafkaClient({
|
|
279
327
|
brokers: 'localhost:29092',
|
|
280
328
|
clientId: 'my-client-id',
|
|
281
329
|
logLevel: 'info',
|
|
282
330
|
brokerAddressFamily: 'v4',
|
|
283
|
-
})
|
|
331
|
+
})
|
|
284
332
|
|
|
285
333
|
// Stream consumer with custom ReadableOptions (v2.0.0+)
|
|
286
|
-
const kafkaStream = kafkaClient.createStreamConsumer(
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
334
|
+
const kafkaStream = kafkaClient.createStreamConsumer(
|
|
335
|
+
{
|
|
336
|
+
groupId: `my-group-id`,
|
|
337
|
+
enableAutoCommit: true,
|
|
338
|
+
},
|
|
339
|
+
{
|
|
340
|
+
objectMode: true, // Default in v2.0.0+
|
|
341
|
+
highWaterMark: 1024,
|
|
342
|
+
encoding: null,
|
|
343
|
+
},
|
|
344
|
+
)
|
|
294
345
|
|
|
295
346
|
await kafkaStream.subscribe([
|
|
296
347
|
{ topic: 'foo', createTopic: true },
|
|
297
|
-
{ topic: 'bar', createTopic: true }
|
|
298
|
-
])
|
|
348
|
+
{ topic: 'bar', createTopic: true },
|
|
349
|
+
])
|
|
299
350
|
|
|
300
351
|
kafkaStream.on('data', (message) => {
|
|
301
352
|
console.log('>>> Message received:', {
|
|
302
353
|
payload: message.payload.toString(),
|
|
303
354
|
offset: message.offset,
|
|
304
355
|
partition: message.partition,
|
|
305
|
-
topic: message.topic
|
|
306
|
-
})
|
|
356
|
+
topic: message.topic,
|
|
357
|
+
})
|
|
307
358
|
|
|
308
359
|
if (message.offset > 10) {
|
|
309
|
-
kafkaStream.destroy()
|
|
360
|
+
kafkaStream.destroy()
|
|
310
361
|
}
|
|
311
|
-
})
|
|
362
|
+
})
|
|
312
363
|
|
|
313
364
|
kafkaStream.on('close', () => {
|
|
314
|
-
kafkaStream.unsubscribe()
|
|
315
|
-
console.log('Stream ended')
|
|
316
|
-
})
|
|
365
|
+
kafkaStream.unsubscribe()
|
|
366
|
+
console.log('Stream ended')
|
|
367
|
+
})
|
|
317
368
|
```
|
|
318
369
|
|
|
319
370
|
## Producer Examples
|
|
@@ -325,35 +376,37 @@ const kafkaClient = new KafkaClient({
|
|
|
325
376
|
brokers: 'localhost:29092',
|
|
326
377
|
clientId: 'my-client-id',
|
|
327
378
|
brokerAddressFamily: 'v4',
|
|
328
|
-
})
|
|
379
|
+
})
|
|
329
380
|
|
|
330
381
|
// Enhanced producer with flexible configuration
|
|
331
382
|
const producer = kafkaClient.createProducer({
|
|
332
383
|
configuration: {
|
|
333
|
-
'batch.size': 50000,
|
|
334
|
-
'linger.ms': 10,
|
|
384
|
+
'batch.size': 50000, // Number value supported
|
|
385
|
+
'linger.ms': 10, // Number value supported
|
|
335
386
|
'compression.type': 'lz4',
|
|
336
|
-
'enable.idempotence': true
|
|
337
|
-
}
|
|
338
|
-
})
|
|
387
|
+
'enable.idempotence': true, // Boolean value supported
|
|
388
|
+
},
|
|
389
|
+
})
|
|
339
390
|
|
|
340
391
|
const messages = Array.from({ length: 100 }, (_, i) => ({
|
|
341
|
-
payload: Buffer.from(
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
})
|
|
392
|
+
payload: Buffer.from(
|
|
393
|
+
JSON.stringify({
|
|
394
|
+
_id: i,
|
|
395
|
+
name: `Batch Message ${i}`,
|
|
396
|
+
timestamp: new Date().toISOString(),
|
|
397
|
+
}),
|
|
398
|
+
),
|
|
399
|
+
}))
|
|
347
400
|
|
|
348
401
|
try {
|
|
349
402
|
const result = await producer.send({
|
|
350
403
|
topic: 'my-topic',
|
|
351
|
-
messages
|
|
352
|
-
})
|
|
353
|
-
console.log('Batch sent. Offset:', result)
|
|
354
|
-
console.assert(result.length === 100)
|
|
404
|
+
messages,
|
|
405
|
+
})
|
|
406
|
+
console.log('Batch sent. Offset:', result)
|
|
407
|
+
console.assert(result.length === 100)
|
|
355
408
|
} catch (error) {
|
|
356
|
-
console.error('Batch error:', error)
|
|
409
|
+
console.error('Batch error:', error)
|
|
357
410
|
}
|
|
358
411
|
```
|
|
359
412
|
|
|
@@ -363,29 +416,33 @@ try {
|
|
|
363
416
|
async function produceWithMetadata() {
|
|
364
417
|
const producer = kafkaClient.createProducer({
|
|
365
418
|
configuration: {
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
'max.in.flight.requests.per.connection': 1
|
|
369
|
-
}
|
|
370
|
-
})
|
|
419
|
+
acks: 'all',
|
|
420
|
+
retries: 5,
|
|
421
|
+
'max.in.flight.requests.per.connection': 1,
|
|
422
|
+
},
|
|
423
|
+
})
|
|
371
424
|
|
|
372
425
|
try {
|
|
373
426
|
await producer.send({
|
|
374
427
|
topic: 'user-events',
|
|
375
|
-
messages: [
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
428
|
+
messages: [
|
|
429
|
+
{
|
|
430
|
+
key: 'user-123',
|
|
431
|
+
payload: Buffer.from(
|
|
432
|
+
JSON.stringify({
|
|
433
|
+
userId: 123,
|
|
434
|
+
action: 'update',
|
|
435
|
+
}),
|
|
436
|
+
),
|
|
437
|
+
headers: {
|
|
438
|
+
'correlation-id': 'txn-123',
|
|
439
|
+
source: 'user-service',
|
|
440
|
+
},
|
|
441
|
+
},
|
|
442
|
+
],
|
|
443
|
+
})
|
|
387
444
|
} catch (error) {
|
|
388
|
-
console.error('Error:', error)
|
|
445
|
+
console.error('Error:', error)
|
|
389
446
|
}
|
|
390
447
|
}
|
|
391
448
|
```
|
|
@@ -403,7 +460,7 @@ const kafkaClient = new KafkaClient({
|
|
|
403
460
|
configuration: {
|
|
404
461
|
'auto.offset.reset': 'earliest',
|
|
405
462
|
'session.timeout.ms': 30000,
|
|
406
|
-
'heartbeat.interval.ms': 10000
|
|
463
|
+
'heartbeat.interval.ms': 10000,
|
|
407
464
|
},
|
|
408
465
|
})
|
|
409
466
|
|
|
@@ -411,13 +468,16 @@ const kafkaClient = new KafkaClient({
|
|
|
411
468
|
* Creates and configures a new Kafka stream consumer
|
|
412
469
|
*/
|
|
413
470
|
async function createConsumer() {
|
|
414
|
-
const kafkaStream = kafkaClient.createStreamConsumer(
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
471
|
+
const kafkaStream = kafkaClient.createStreamConsumer(
|
|
472
|
+
{
|
|
473
|
+
groupId: 'reconnect-test',
|
|
474
|
+
enableAutoCommit: true,
|
|
475
|
+
},
|
|
476
|
+
{
|
|
477
|
+
highWaterMark: 100,
|
|
478
|
+
objectMode: true,
|
|
479
|
+
},
|
|
480
|
+
)
|
|
421
481
|
|
|
422
482
|
await kafkaStream.subscribe([
|
|
423
483
|
{ topic: 'foo', createTopic: true },
|
|
@@ -499,65 +559,65 @@ You can find some examples on the [example](https://github.com/flash-tecnologia/
|
|
|
499
559
|
|
|
500
560
|
### KafkaConfiguration
|
|
501
561
|
|
|
502
|
-
| Property
|
|
503
|
-
|
|
|
504
|
-
| `brokers`
|
|
505
|
-
| `clientId`
|
|
506
|
-
| `securityProtocol`
|
|
507
|
-
| `logLevel`
|
|
508
|
-
| `brokerAddressFamily` | `string`
|
|
509
|
-
| `configuration`
|
|
510
|
-
| `diagnostics`
|
|
562
|
+
| Property | Type | Default | Description |
|
|
563
|
+
| --------------------- | --------------------- | ----------- | ------------------------------------------------------------------- |
|
|
564
|
+
| `brokers` | `string` | | List of brokers to connect to |
|
|
565
|
+
| `clientId` | `string` | `"rdkafka"` | Client id to use for the connection |
|
|
566
|
+
| `securityProtocol` | `SecurityProtocol` | | Security protocol to use (PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL) |
|
|
567
|
+
| `logLevel` | `string` | `info` | Log level for the client |
|
|
568
|
+
| `brokerAddressFamily` | `string` | `"v4"` | Address family to use for the connection (v4, v6) |
|
|
569
|
+
| `configuration` | `Record<string, any>` | `{}` | Additional configuration options for the client |
|
|
570
|
+
| `diagnostics` | `boolean` | `false` | **v3.0.0+**: Enable diagnostics channel for OTEL instrumentation |
|
|
511
571
|
|
|
512
572
|
### ConsumerConfiguration
|
|
513
573
|
|
|
514
|
-
| Property
|
|
515
|
-
|
|
|
516
|
-
| `groupId`
|
|
517
|
-
| `enableAutoCommit`
|
|
518
|
-
| `configuration`
|
|
519
|
-
| `fetchMetadataTimeout` | `number`
|
|
520
|
-
| `maxBatchMessages`
|
|
574
|
+
| Property | Type | Default | Description |
|
|
575
|
+
| ---------------------- | --------------------- | ------- | ----------------------------------------- |
|
|
576
|
+
| `groupId` | `string` | | Consumer group ID |
|
|
577
|
+
| `enableAutoCommit` | `boolean` | `true` | Enable automatic offset commits |
|
|
578
|
+
| `configuration` | `Record<string, any>` | `{}` | Additional consumer configuration options |
|
|
579
|
+
| `fetchMetadataTimeout` | `number` | `60000` | Timeout for fetching metadata (ms) |
|
|
580
|
+
| `maxBatchMessages` | `number` | `1000` | Maximum messages in a batch operation |
|
|
521
581
|
|
|
522
582
|
### Consumer Commit Methods
|
|
523
583
|
|
|
524
584
|
kafka-crab-js provides two methods for committing offsets:
|
|
525
585
|
|
|
526
|
-
| Method
|
|
527
|
-
|
|
|
528
|
-
| `commit`
|
|
529
|
-
| `commitMessage` | `commitMessage(message, mode)`
|
|
586
|
+
| Method | Signature | Description |
|
|
587
|
+
| --------------- | ---------------------------------------- | ----------------------------------------------------------------------- |
|
|
588
|
+
| `commit` | `commit(topic, partition, offset, mode)` | Traditional commit - you must calculate `offset + 1` |
|
|
589
|
+
| `commitMessage` | `commitMessage(message, mode)` | **v2.1.0+**: Simplified commit - automatically handles offset increment |
|
|
530
590
|
|
|
531
591
|
```javascript
|
|
532
592
|
// Using commitMessage (recommended for v2.1.0+)
|
|
533
|
-
const message = await consumer.recv()
|
|
534
|
-
await consumer.commitMessage(message, 'Sync')
|
|
593
|
+
const message = await consumer.recv()
|
|
594
|
+
await consumer.commitMessage(message, 'Sync')
|
|
535
595
|
|
|
536
596
|
// Using commit (traditional)
|
|
537
|
-
const message = await consumer.recv()
|
|
538
|
-
await consumer.commit(message.topic, message.partition, message.offset + 1, 'Sync')
|
|
597
|
+
const message = await consumer.recv()
|
|
598
|
+
await consumer.commit(message.topic, message.partition, message.offset + 1, 'Sync')
|
|
539
599
|
```
|
|
540
600
|
|
|
541
601
|
Both methods support `'Sync'` and `'Async'` commit modes.
|
|
542
602
|
|
|
543
603
|
### ProducerConfiguration
|
|
544
604
|
|
|
545
|
-
| Property
|
|
546
|
-
|
|
|
547
|
-
| `queueTimeout`
|
|
548
|
-
| `autoFlush`
|
|
549
|
-
| `configuration` | `Record<string, any>` | `{}`
|
|
605
|
+
| Property | Type | Default | Description |
|
|
606
|
+
| --------------- | --------------------- | ------- | ----------------------------------------- |
|
|
607
|
+
| `queueTimeout` | `number` | `5000` | Queue timeout in milliseconds |
|
|
608
|
+
| `autoFlush` | `boolean` | `true` | Enable automatic message flushing |
|
|
609
|
+
| `configuration` | `Record<string, any>` | `{}` | Additional producer configuration options |
|
|
550
610
|
|
|
551
611
|
### TopicPartitionConfig
|
|
552
612
|
|
|
553
|
-
| Property
|
|
554
|
-
|
|
|
555
|
-
| `topic`
|
|
556
|
-
| `allOffsets`
|
|
557
|
-
| `partitionOffset` | `Array<PartitionOffset>`
|
|
558
|
-
| `createTopic`
|
|
559
|
-
| `numPartitions`
|
|
560
|
-
| `replicas`
|
|
613
|
+
| Property | Type | Default | Description |
|
|
614
|
+
| ----------------- | ------------------------ | ------- | ----------------------------------------------------- |
|
|
615
|
+
| `topic` | `string` | | Topic name |
|
|
616
|
+
| `allOffsets` | `OffsetModel` | | Offset configuration for all partitions |
|
|
617
|
+
| `partitionOffset` | `Array<PartitionOffset>` | | Per-partition offset configuration |
|
|
618
|
+
| `createTopic` | `boolean` | `false` | **v2.0.0+**: Create topic if it doesn't exist |
|
|
619
|
+
| `numPartitions` | `number` | `1` | **v2.0.0+**: Number of partitions when creating topic |
|
|
620
|
+
| `replicas` | `number` | `1` | **v2.0.0+**: Number of replicas when creating topic |
|
|
561
621
|
|
|
562
622
|
You can see the available options here: [librdkafka](https://docs.confluent.io/platform/current/clients/librdkafka/html/md_CONFIGURATION.html).
|
|
563
623
|
|
|
@@ -601,6 +661,7 @@ For complete configuration options and examples, see the [kafka-crab-js-otel REA
|
|
|
601
661
|
#### Examples
|
|
602
662
|
|
|
603
663
|
See comprehensive examples in the `example/` directory:
|
|
664
|
+
|
|
604
665
|
- `example/otel-tracing-example.mjs` - Complete tracing setup with Jaeger
|
|
605
666
|
- `example/otel-metrics-example.mjs` - Metrics collection with Prometheus
|
|
606
667
|
- `example/README.md` - Full documentation for all examples
|
|
@@ -612,17 +673,19 @@ See comprehensive examples in the `example/` directory:
|
|
|
612
673
|
kafka-crab-js v2.0.0+ includes a comprehensive benchmark suite to compare performance against other popular Kafka clients:
|
|
613
674
|
|
|
614
675
|
```bash
|
|
676
|
+
cd ../benchmark
|
|
677
|
+
|
|
615
678
|
# Set up benchmark environment (requires Kafka running locally)
|
|
616
|
-
|
|
617
|
-
|
|
679
|
+
vp install
|
|
680
|
+
vp run setup:consumer
|
|
618
681
|
|
|
619
682
|
# Run consumer performance benchmarks
|
|
620
|
-
|
|
683
|
+
vp run benchmark
|
|
621
684
|
```
|
|
622
685
|
|
|
623
686
|
### Benchmark Results
|
|
624
687
|
|
|
625
|
-
|
|
688
|
+
_Benchmarks run on macOS with Apple M1 chip processing 50,000 messages (December 2024)_
|
|
626
689
|
|
|
627
690
|
```
|
|
628
691
|
╔════════════════════════╤═════════╤══════════════════╤═══════════╤══════════════════════════╗
|
|
@@ -640,6 +703,7 @@ node benchmark/consumer.ts
|
|
|
640
703
|
```
|
|
641
704
|
|
|
642
705
|
The benchmark suite compares:
|
|
706
|
+
|
|
643
707
|
- **kafka-crab-js (serial)**: Single message processing - **43,214 ops/sec**
|
|
644
708
|
- **kafka-crab-js (batch)**: Batch message processing - **205,985 ops/sec** (fastest)
|
|
645
709
|
- **node-rdkafka (evented)**: Event-based processing - **24,923 ops/sec**
|
|
@@ -647,6 +711,7 @@ The benchmark suite compares:
|
|
|
647
711
|
- **kafkajs**: Official KafkaJS client - **834 ops/sec**
|
|
648
712
|
|
|
649
713
|
Performance characteristics:
|
|
714
|
+
|
|
650
715
|
- **52x faster than kafkajs** in serial mode, **247x faster in batch mode**
|
|
651
716
|
- **High throughput**: Batch processing provides 4.8x performance improvement over serial mode
|
|
652
717
|
- **Low latency**: Optimized for both single and batch message processing
|
|
@@ -662,41 +727,46 @@ export const topic = 'benchmarks'
|
|
|
662
727
|
export const brokers = ['localhost:9092', 'localhost:9093', 'localhost:9094']
|
|
663
728
|
|
|
664
729
|
// Benchmark parameters can be adjusted in consumer.ts:
|
|
665
|
-
const iterations = 10_000
|
|
666
|
-
const maxBytes = 200
|
|
730
|
+
const iterations = 10_000 // Number of messages to process
|
|
731
|
+
const maxBytes = 200 // Maximum message size
|
|
667
732
|
```
|
|
668
733
|
|
|
669
734
|
## Best Practices
|
|
670
735
|
|
|
671
736
|
### Error Handling
|
|
737
|
+
|
|
672
738
|
- Always wrap async operations in try-catch blocks
|
|
673
739
|
- Implement proper error logging and monitoring
|
|
674
740
|
- Handle both operational and programming errors separately
|
|
675
741
|
|
|
676
742
|
### Performance
|
|
743
|
+
|
|
677
744
|
- Use batch operations for high-throughput scenarios
|
|
678
745
|
- Configure appropriate batch sizes and compression settings
|
|
679
746
|
- Monitor and tune consumer group performance
|
|
680
747
|
- Leverage the benchmark suite to optimize your specific use case
|
|
681
748
|
|
|
682
749
|
### Configuration (v2.0.0+)
|
|
750
|
+
|
|
683
751
|
- Use the flexible configuration system with proper data types:
|
|
684
752
|
```javascript
|
|
685
753
|
const config = {
|
|
686
|
-
'batch.size': 16384,
|
|
687
|
-
'compression.type': 'snappy',
|
|
688
|
-
'enable.idempotence': true,
|
|
689
|
-
|
|
754
|
+
'batch.size': 16384, // number
|
|
755
|
+
'compression.type': 'snappy', // string
|
|
756
|
+
'enable.idempotence': true, // boolean
|
|
757
|
+
retries: 5, // number
|
|
690
758
|
}
|
|
691
759
|
```
|
|
692
760
|
|
|
693
761
|
### Message Processing
|
|
762
|
+
|
|
694
763
|
- Validate message formats before processing
|
|
695
764
|
- Implement proper serialization/deserialization
|
|
696
765
|
- Handle message ordering when required
|
|
697
766
|
- Use topic creation options for better topic management
|
|
698
767
|
|
|
699
768
|
### Stream Processing (v2.0.0+)
|
|
769
|
+
|
|
700
770
|
- Configure appropriate `ReadableOptions` for your use case
|
|
701
771
|
- Use `objectMode: true` for structured message processing
|
|
702
772
|
- Set appropriate `highWaterMark` based on memory constraints
|
|
@@ -714,20 +784,17 @@ const maxBytes = 200 // Maximum message size
|
|
|
714
784
|
|
|
715
785
|
```bash
|
|
716
786
|
# Build the project
|
|
717
|
-
|
|
787
|
+
vp run build
|
|
718
788
|
|
|
719
789
|
# Run type checking
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
# Run linting
|
|
723
|
-
pnpm lint
|
|
790
|
+
vp run typecheck
|
|
724
791
|
|
|
725
|
-
#
|
|
726
|
-
|
|
792
|
+
# Run linting and formatting
|
|
793
|
+
vp check
|
|
727
794
|
|
|
728
795
|
# Run benchmarks
|
|
729
|
-
|
|
730
|
-
|
|
796
|
+
cd ../benchmark
|
|
797
|
+
vp run benchmark
|
|
731
798
|
```
|
|
732
799
|
|
|
733
800
|
## OpenTelemetry Instrumentation
|
|
@@ -781,7 +848,7 @@ context.setGlobalContextManager(new AsyncHooksContextManager().enable())
|
|
|
781
848
|
enableOtelInstrumentation({
|
|
782
849
|
captureMessagePayload: true,
|
|
783
850
|
captureMessageHeaders: true,
|
|
784
|
-
producerHook: span => span.setAttribute('messaging.client.kind', 'producer'),
|
|
851
|
+
producerHook: (span) => span.setAttribute('messaging.client.kind', 'producer'),
|
|
785
852
|
})
|
|
786
853
|
|
|
787
854
|
// 3. Create client with diagnostics enabled
|
|
@@ -795,10 +862,12 @@ const client = new KafkaClient({
|
|
|
795
862
|
const producer = client.createProducer()
|
|
796
863
|
await producer.send({
|
|
797
864
|
topic: 'orders',
|
|
798
|
-
messages: [
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
865
|
+
messages: [
|
|
866
|
+
{
|
|
867
|
+
payload: Buffer.from(JSON.stringify({ orderId: '123' })),
|
|
868
|
+
headers: { 'custom-header': 'foo' },
|
|
869
|
+
},
|
|
870
|
+
],
|
|
802
871
|
})
|
|
803
872
|
|
|
804
873
|
// 5. Use stream consumer with proper cleanup
|
|
@@ -809,7 +878,7 @@ const consumer = client.createStreamConsumer({
|
|
|
809
878
|
|
|
810
879
|
await consumer.subscribe('orders')
|
|
811
880
|
|
|
812
|
-
consumer.on('data', message => {
|
|
881
|
+
consumer.on('data', (message) => {
|
|
813
882
|
try {
|
|
814
883
|
console.log(message.headers?.['custom-header']?.toString())
|
|
815
884
|
} finally {
|