kafka-crab-js 3.1.0 → 4.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +239 -170
  2. package/dist/base-kafka-stream-readable-BokUK06Z.d.ts +69 -0
  3. package/dist/base-kafka-stream-readable-I1xda63L.d.cts +69 -0
  4. package/dist/channels-CYQ4ni8K.d.cts +133 -0
  5. package/dist/channels-QBobKRC2.d.ts +133 -0
  6. package/dist/diagnostics/channels.cjs +85 -0
  7. package/dist/diagnostics/channels.cjs.map +1 -0
  8. package/dist/diagnostics/channels.d.cts +2 -0
  9. package/dist/diagnostics/channels.d.ts +2 -138
  10. package/dist/diagnostics/channels.js +72 -0
  11. package/dist/diagnostics/channels.js.map +1 -0
  12. package/dist/diagnostics/index.cjs +21 -0
  13. package/dist/diagnostics/index.d.cts +3 -0
  14. package/dist/diagnostics/index.d.ts +3 -7
  15. package/dist/diagnostics/index.js +3 -0
  16. package/dist/diagnostics/instrumentation.cjs +334 -0
  17. package/dist/diagnostics/instrumentation.cjs.map +1 -0
  18. package/dist/diagnostics/instrumentation.d.cts +2 -0
  19. package/dist/diagnostics/instrumentation.d.ts +2 -31
  20. package/dist/diagnostics/instrumentation.js +329 -0
  21. package/dist/diagnostics/instrumentation.js.map +1 -0
  22. package/dist/index.cjs +47 -701
  23. package/dist/index.d.cts +8 -0
  24. package/dist/index.d.ts +8 -8
  25. package/dist/index.js +9 -662
  26. package/dist/instrumentation-BiIPAe1B.d.cts +37 -0
  27. package/dist/instrumentation-CwoJPHwZ.d.ts +37 -0
  28. package/dist/kafka-client.cjs +314 -0
  29. package/dist/kafka-client.cjs.map +1 -0
  30. package/dist/kafka-client.d.cts +107 -0
  31. package/dist/kafka-client.d.ts +98 -52
  32. package/dist/kafka-client.js +313 -0
  33. package/dist/kafka-client.js.map +1 -0
  34. package/dist/streams/base-kafka-stream-readable.cjs +109 -0
  35. package/dist/streams/base-kafka-stream-readable.cjs.map +1 -0
  36. package/dist/streams/base-kafka-stream-readable.d.cts +2 -0
  37. package/dist/streams/base-kafka-stream-readable.d.ts +2 -60
  38. package/dist/streams/base-kafka-stream-readable.js +108 -0
  39. package/dist/streams/base-kafka-stream-readable.js.map +1 -0
  40. package/dist/streams/kafka-batch-stream-readable.cjs +106 -0
  41. package/dist/streams/kafka-batch-stream-readable.cjs.map +1 -0
  42. package/dist/streams/kafka-batch-stream-readable.d.cts +46 -0
  43. package/dist/streams/kafka-batch-stream-readable.d.ts +40 -25
  44. package/dist/streams/kafka-batch-stream-readable.js +105 -0
  45. package/dist/streams/kafka-batch-stream-readable.js.map +1 -0
  46. package/dist/streams/kafka-stream-readable.cjs +63 -0
  47. package/dist/streams/kafka-stream-readable.cjs.map +1 -0
  48. package/dist/streams/kafka-stream-readable.d.cts +27 -0
  49. package/dist/streams/kafka-stream-readable.d.ts +22 -7
  50. package/dist/streams/kafka-stream-readable.js +62 -0
  51. package/dist/streams/kafka-stream-readable.js.map +1 -0
  52. package/js-binding.cjs +52 -52
  53. package/js-binding.d.ts +32 -0
  54. package/js-binding.js +52 -52
  55. package/package.json +53 -52
  56. package/dist/index.cjs.map +0 -7
  57. package/dist/index.js.map +0 -7
package/README.md CHANGED
@@ -6,10 +6,51 @@ A lightweight, flexible, and reliable Kafka client for JavaScript/TypeScript. It
6
6
 
7
7
  [![npm version](https://img.shields.io/npm/v/kafka-crab-js.svg)](https://www.npmjs.com/package/kafka-crab-js)
8
8
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
9
+
9
10
  </div>
10
11
 
11
12
  ---
12
13
 
14
+ ## What's New in Version 4.0.0
15
+
16
+ ### BREAKING CHANGES ⚠️
17
+
18
+ **`CommitMode`, `KafkaEventName`, `PartitionPosition`, and `SecurityProtocol` are no longer runtime exports from `kafka-crab-js`.**
19
+
20
+ These names are now type-only at the root package API. This change applies to the `kafka-crab-js` root import surface and is intended as a major-version change.
21
+
22
+ #### Migration from v3.x
23
+
24
+ **Before (v3.x):**
25
+
26
+ ```ts
27
+ import { CommitMode, SecurityProtocol } from 'kafka-crab-js'
28
+ ```
29
+
30
+ **After for type-only usage:**
31
+
32
+ ```ts
33
+ import type { CommitMode, SecurityProtocol } from 'kafka-crab-js'
34
+ ```
35
+
36
+ **After for runtime usage:**
37
+
38
+ ```ts
39
+ const commitMode: 'Sync' | 'Async' = 'Sync'
40
+ const securityProtocol: 'Plaintext' | 'Ssl' | 'SaslPlaintext' | 'SaslSsl' = 'Plaintext'
41
+ const eventName: 'PreRebalance' | 'PostRebalance' | 'CommitCallback' = 'PreRebalance'
42
+ const partitionPosition: 'Beginning' | 'End' | 'Stored' | 'Invalid' = 'Beginning'
43
+ ```
44
+
45
+ #### Replacement Values
46
+
47
+ | Name | Replacement string literal values |
48
+ | ------------------- | ------------------------------------------------------- |
49
+ | `CommitMode` | `'Sync' \| 'Async'` |
50
+ | `KafkaEventName` | `'PreRebalance' \| 'PostRebalance' \| 'CommitCallback'` |
51
+ | `PartitionPosition` | `'Beginning' \| 'End' \| 'Stored' \| 'Invalid'` |
52
+ | `SecurityProtocol` | `'Plaintext' \| 'Ssl' \| 'SaslPlaintext' \| 'SaslSsl'` |
53
+
13
54
  ## What's New in Version 3.0.0
14
55
 
15
56
  ### BREAKING CHANGES ⚠️
@@ -21,6 +62,7 @@ This change reduces the core package size and makes OTEL an opt-in dependency.
21
62
  #### Migration from v2.x
22
63
 
23
64
  **Before (v2.x):**
65
+
24
66
  ```javascript
25
67
  import { KafkaClient } from 'kafka-crab-js'
26
68
 
@@ -35,6 +77,7 @@ const client = new KafkaClient({
35
77
  ```
36
78
 
37
79
  **After (v3.x):**
80
+
38
81
  ```javascript
39
82
  import { KafkaClient } from 'kafka-crab-js'
40
83
  import { enableOtelInstrumentation, endSpan } from 'kafka-crab-js-otel'
@@ -60,21 +103,22 @@ endSpan(message)
60
103
 
61
104
  #### Key Changes
62
105
 
63
- | v2.x | v3.x |
64
- |------|------|
106
+ | v2.x | v3.x |
107
+ | ---------------------------- | ------------------------------------------------------- |
65
108
  | `otel` config in KafkaClient | `enableOtelInstrumentation()` from `kafka-crab-js-otel` |
66
- | Automatic span ending | Call `endSpan(message)` manually for consumers |
67
- | N/A | `diagnostics: true` required in KafkaClient config |
68
- | OTEL bundled | Install `kafka-crab-js-otel` separately |
109
+ | Automatic span ending | Call `endSpan(message)` manually for consumers |
110
+ | N/A | `diagnostics: true` required in KafkaClient config |
111
+ | OTEL bundled | Install `kafka-crab-js-otel` separately |
69
112
 
70
113
  #### New Package Structure
71
114
 
72
- | Package | Description |
73
- |---------|-------------|
74
- | `kafka-crab-js` | Core Kafka client (producer, consumer, streams) |
115
+ | Package | Description |
116
+ | -------------------- | ------------------------------------------------ |
117
+ | `kafka-crab-js` | Core Kafka client (producer, consumer, streams) |
75
118
  | `kafka-crab-js-otel` | OpenTelemetry instrumentation (separate install) |
76
119
 
77
120
  ### Internal Improvements
121
+
78
122
  - Uses Node.js `diagnostics_channel` for observability (zero overhead when not subscribed)
79
123
  - Cleaner separation of concerns between core and observability
80
124
  - Smaller bundle size for users who don't need OTEL
@@ -94,13 +138,13 @@ endSpan(message)
94
138
  - Available on both `KafkaConsumer` and stream consumers
95
139
  - **Before** (v2.0.0):
96
140
  ```javascript
97
- const message = await consumer.recv();
98
- await consumer.commit(message.topic, message.partition, message.offset + 1, 'Sync');
141
+ const message = await consumer.recv()
142
+ await consumer.commit(message.topic, message.partition, message.offset + 1, 'Sync')
99
143
  ```
100
144
  - **After** (v2.1.0):
101
145
  ```javascript
102
- const message = await consumer.recv();
103
- await consumer.commitMessage(message, 'Sync');
146
+ const message = await consumer.recv()
147
+ await consumer.commitMessage(message, 'Sync')
104
148
  ```
105
149
 
106
150
  2. **Enhanced OpenTelemetry Support**:
@@ -179,14 +223,14 @@ pnpm add kafka-crab-js
179
223
  ### Basic Consumer Setup
180
224
 
181
225
  ```javascript
182
- import { KafkaClient } from 'kafka-crab-js';
226
+ import { KafkaClient } from 'kafka-crab-js'
183
227
  async function run() {
184
228
  const kafkaClient = new KafkaClient({
185
229
  brokers: 'localhost:29092',
186
230
  clientId: 'foo-client',
187
231
  logLevel: 'debug',
188
232
  brokerAddressFamily: 'v4',
189
- });
233
+ })
190
234
 
191
235
  // Create consumer with topic creation control
192
236
  const consumer = kafkaClient.createConsumer({
@@ -194,77 +238,81 @@ async function run() {
194
238
  configuration: {
195
239
  'auto.offset.reset': 'earliest',
196
240
  'enable.auto.commit': false, // Use manual commit for better control
197
- }
198
- });
241
+ },
242
+ })
199
243
 
200
244
  // Subscribe with topic creation options
201
- await consumer.subscribe([{
202
- topic: 'foo',
203
- createTopic: true,
204
- numPartitions: 3
205
- }]);
206
-
207
- const message = await consumer.recv();
208
- const { payload, partition, offset, topic } = message;
245
+ await consumer.subscribe([
246
+ {
247
+ topic: 'foo',
248
+ createTopic: true,
249
+ numPartitions: 3,
250
+ },
251
+ ])
252
+
253
+ const message = await consumer.recv()
254
+ const { payload, partition, offset, topic } = message
209
255
  console.log({
210
256
  topic,
211
257
  partition,
212
258
  offset,
213
- value: payload.toString()
214
- });
259
+ value: payload.toString(),
260
+ })
215
261
 
216
262
  // Manual commit - two options:
217
263
  // Option 1 (v2.1.0+): Simplified with commitMessage
218
- await consumer.commitMessage(message, 'Sync');
219
-
264
+ await consumer.commitMessage(message, 'Sync')
265
+
220
266
  // Option 2: Traditional commit with manual offset increment
221
267
  // await consumer.commit(topic, partition, offset + 1, 'Sync');
222
268
 
223
- consumer.unsubscribe();
269
+ consumer.unsubscribe()
224
270
  }
225
271
 
226
- await run();
272
+ await run()
227
273
  ```
228
274
 
229
275
  ### Basic Producer Setup
230
276
 
231
277
  ```javascript
232
- import { KafkaClient } from 'kafka-crab-js';
278
+ import { KafkaClient } from 'kafka-crab-js'
233
279
 
234
280
  const kafkaClient = new KafkaClient({
235
281
  brokers: 'localhost:29092',
236
282
  clientId: 'my-client-id',
237
283
  logLevel: 'info',
238
284
  brokerAddressFamily: 'v4',
239
- });
285
+ })
240
286
 
241
287
  // Producer configuration is now optional with sensible defaults
242
288
  const producer = kafkaClient.createProducer({
243
289
  configuration: {
244
- 'message.timeout.ms': 5000, // Now supports number values
290
+ 'message.timeout.ms': 5000, // Now supports number values
245
291
  'batch.size': 16384,
246
- 'compression.type': 'snappy'
247
- }
248
- });
292
+ 'compression.type': 'snappy',
293
+ },
294
+ })
249
295
 
250
296
  const message = {
251
297
  id: 1,
252
- name: "Sample Message",
253
- timestamp: new Date().toISOString()
254
- };
298
+ name: 'Sample Message',
299
+ timestamp: new Date().toISOString(),
300
+ }
255
301
 
256
302
  const result = await producer.send({
257
303
  topic: 'my-topic',
258
- messages: [{
259
- payload: Buffer.from(JSON.stringify(message))
260
- }]
261
- });
304
+ messages: [
305
+ {
306
+ payload: Buffer.from(JSON.stringify(message)),
307
+ },
308
+ ],
309
+ })
262
310
 
263
- const errors = result.map(r => r.error).filter(Boolean);
311
+ const errors = result.map((r) => r.error).filter(Boolean)
264
312
  if (errors.length > 0) {
265
- console.error('Error sending message:', errors);
313
+ console.error('Error sending message:', errors)
266
314
  } else {
267
- console.log('Message sent. Offset:', result);
315
+ console.log('Message sent. Offset:', result)
268
316
  }
269
317
  ```
270
318
 
@@ -273,47 +321,50 @@ if (errors.length > 0) {
273
321
  ### Enhanced Stream Consumer Example
274
322
 
275
323
  ```javascript
276
- import { KafkaClient } from 'kafka-crab-js';
324
+ import { KafkaClient } from 'kafka-crab-js'
277
325
 
278
326
  const kafkaClient = new KafkaClient({
279
327
  brokers: 'localhost:29092',
280
328
  clientId: 'my-client-id',
281
329
  logLevel: 'info',
282
330
  brokerAddressFamily: 'v4',
283
- });
331
+ })
284
332
 
285
333
  // Stream consumer with custom ReadableOptions (v2.0.0+)
286
- const kafkaStream = kafkaClient.createStreamConsumer({
287
- groupId: `my-group-id`,
288
- enableAutoCommit: true,
289
- }, {
290
- objectMode: true, // Default in v2.0.0+
291
- highWaterMark: 1024,
292
- encoding: null
293
- });
334
+ const kafkaStream = kafkaClient.createStreamConsumer(
335
+ {
336
+ groupId: `my-group-id`,
337
+ enableAutoCommit: true,
338
+ },
339
+ {
340
+ objectMode: true, // Default in v2.0.0+
341
+ highWaterMark: 1024,
342
+ encoding: null,
343
+ },
344
+ )
294
345
 
295
346
  await kafkaStream.subscribe([
296
347
  { topic: 'foo', createTopic: true },
297
- { topic: 'bar', createTopic: true }
298
- ]);
348
+ { topic: 'bar', createTopic: true },
349
+ ])
299
350
 
300
351
  kafkaStream.on('data', (message) => {
301
352
  console.log('>>> Message received:', {
302
353
  payload: message.payload.toString(),
303
354
  offset: message.offset,
304
355
  partition: message.partition,
305
- topic: message.topic
306
- });
356
+ topic: message.topic,
357
+ })
307
358
 
308
359
  if (message.offset > 10) {
309
- kafkaStream.destroy();
360
+ kafkaStream.destroy()
310
361
  }
311
- });
362
+ })
312
363
 
313
364
  kafkaStream.on('close', () => {
314
- kafkaStream.unsubscribe();
315
- console.log('Stream ended');
316
- });
365
+ kafkaStream.unsubscribe()
366
+ console.log('Stream ended')
367
+ })
317
368
  ```
318
369
 
319
370
  ## Producer Examples
@@ -325,35 +376,37 @@ const kafkaClient = new KafkaClient({
325
376
  brokers: 'localhost:29092',
326
377
  clientId: 'my-client-id',
327
378
  brokerAddressFamily: 'v4',
328
- });
379
+ })
329
380
 
330
381
  // Enhanced producer with flexible configuration
331
382
  const producer = kafkaClient.createProducer({
332
383
  configuration: {
333
- 'batch.size': 50000, // Number value supported
334
- 'linger.ms': 10, // Number value supported
384
+ 'batch.size': 50000, // Number value supported
385
+ 'linger.ms': 10, // Number value supported
335
386
  'compression.type': 'lz4',
336
- 'enable.idempotence': true // Boolean value supported
337
- }
338
- });
387
+ 'enable.idempotence': true, // Boolean value supported
388
+ },
389
+ })
339
390
 
340
391
  const messages = Array.from({ length: 100 }, (_, i) => ({
341
- payload: Buffer.from(JSON.stringify({
342
- _id: i,
343
- name: `Batch Message ${i}`,
344
- timestamp: new Date().toISOString()
345
- }))
346
- }));
392
+ payload: Buffer.from(
393
+ JSON.stringify({
394
+ _id: i,
395
+ name: `Batch Message ${i}`,
396
+ timestamp: new Date().toISOString(),
397
+ }),
398
+ ),
399
+ }))
347
400
 
348
401
  try {
349
402
  const result = await producer.send({
350
403
  topic: 'my-topic',
351
- messages
352
- });
353
- console.log('Batch sent. Offset:', result);
354
- console.assert(result.length === 100);
404
+ messages,
405
+ })
406
+ console.log('Batch sent. Offset:', result)
407
+ console.assert(result.length === 100)
355
408
  } catch (error) {
356
- console.error('Batch error:', error);
409
+ console.error('Batch error:', error)
357
410
  }
358
411
  ```
359
412
 
@@ -363,29 +416,33 @@ try {
363
416
  async function produceWithMetadata() {
364
417
  const producer = kafkaClient.createProducer({
365
418
  configuration: {
366
- 'acks': 'all',
367
- 'retries': 5,
368
- 'max.in.flight.requests.per.connection': 1
369
- }
370
- });
419
+ acks: 'all',
420
+ retries: 5,
421
+ 'max.in.flight.requests.per.connection': 1,
422
+ },
423
+ })
371
424
 
372
425
  try {
373
426
  await producer.send({
374
427
  topic: 'user-events',
375
- messages: [{
376
- key: 'user-123',
377
- payload: Buffer.from(JSON.stringify({
378
- userId: 123,
379
- action: 'update'
380
- })),
381
- headers: {
382
- 'correlation-id': 'txn-123',
383
- 'source': 'user-service'
384
- }
385
- }]
386
- });
428
+ messages: [
429
+ {
430
+ key: 'user-123',
431
+ payload: Buffer.from(
432
+ JSON.stringify({
433
+ userId: 123,
434
+ action: 'update',
435
+ }),
436
+ ),
437
+ headers: {
438
+ 'correlation-id': 'txn-123',
439
+ source: 'user-service',
440
+ },
441
+ },
442
+ ],
443
+ })
387
444
  } catch (error) {
388
- console.error('Error:', error);
445
+ console.error('Error:', error)
389
446
  }
390
447
  }
391
448
  ```
@@ -403,7 +460,7 @@ const kafkaClient = new KafkaClient({
403
460
  configuration: {
404
461
  'auto.offset.reset': 'earliest',
405
462
  'session.timeout.ms': 30000,
406
- 'heartbeat.interval.ms': 10000
463
+ 'heartbeat.interval.ms': 10000,
407
464
  },
408
465
  })
409
466
 
@@ -411,13 +468,16 @@ const kafkaClient = new KafkaClient({
411
468
  * Creates and configures a new Kafka stream consumer
412
469
  */
413
470
  async function createConsumer() {
414
- const kafkaStream = kafkaClient.createStreamConsumer({
415
- groupId: 'reconnect-test',
416
- enableAutoCommit: true,
417
- }, {
418
- highWaterMark: 100,
419
- objectMode: true
420
- })
471
+ const kafkaStream = kafkaClient.createStreamConsumer(
472
+ {
473
+ groupId: 'reconnect-test',
474
+ enableAutoCommit: true,
475
+ },
476
+ {
477
+ highWaterMark: 100,
478
+ objectMode: true,
479
+ },
480
+ )
421
481
 
422
482
  await kafkaStream.subscribe([
423
483
  { topic: 'foo', createTopic: true },
@@ -499,65 +559,65 @@ You can find some examples on the [example](https://github.com/flash-tecnologia/
499
559
 
500
560
  ### KafkaConfiguration
501
561
 
502
- | Property | Type | Default | Description |
503
- | --- | --- | --- | --- |
504
- | `brokers` | `string` || List of brokers to connect to |
505
- | `clientId` | `string` | `"rdkafka"` | Client id to use for the connection |
506
- | `securityProtocol` | `SecurityProtocol` || Security protocol to use (PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL) |
507
- | `logLevel` | `string` | `info` | Log level for the client |
508
- | `brokerAddressFamily` | `string` | `"v4"` | Address family to use for the connection (v4, v6) |
509
- | `configuration` | `Record<string, any>` | `{}` | Additional configuration options for the client |
510
- | `diagnostics` | `boolean` | `false` | **v3.0.0+**: Enable diagnostics channel for OTEL instrumentation |
562
+ | Property | Type | Default | Description |
563
+ | --------------------- | --------------------- | ----------- | ------------------------------------------------------------------- |
564
+ | `brokers` | `string` | | List of brokers to connect to |
565
+ | `clientId` | `string` | `"rdkafka"` | Client id to use for the connection |
566
+ | `securityProtocol` | `SecurityProtocol` | | Security protocol to use (PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL) |
567
+ | `logLevel` | `string` | `info` | Log level for the client |
568
+ | `brokerAddressFamily` | `string` | `"v4"` | Address family to use for the connection (v4, v6) |
569
+ | `configuration` | `Record<string, any>` | `{}` | Additional configuration options for the client |
570
+ | `diagnostics` | `boolean` | `false` | **v3.0.0+**: Enable diagnostics channel for OTEL instrumentation |
511
571
 
512
572
  ### ConsumerConfiguration
513
573
 
514
- | Property | Type | Default | Description |
515
- | --- | --- | --- | --- |
516
- | `groupId` | `string` || Consumer group ID |
517
- | `enableAutoCommit` | `boolean` | `true` | Enable automatic offset commits |
518
- | `configuration` | `Record<string, any>` | `{}` | Additional consumer configuration options |
519
- | `fetchMetadataTimeout` | `number` | `60000` | Timeout for fetching metadata (ms) |
520
- | `maxBatchMessages` | `number` | `1000` | Maximum messages in a batch operation |
574
+ | Property | Type | Default | Description |
575
+ | ---------------------- | --------------------- | ------- | ----------------------------------------- |
576
+ | `groupId` | `string` | | Consumer group ID |
577
+ | `enableAutoCommit` | `boolean` | `true` | Enable automatic offset commits |
578
+ | `configuration` | `Record<string, any>` | `{}` | Additional consumer configuration options |
579
+ | `fetchMetadataTimeout` | `number` | `60000` | Timeout for fetching metadata (ms) |
580
+ | `maxBatchMessages` | `number` | `1000` | Maximum messages in a batch operation |
521
581
 
522
582
  ### Consumer Commit Methods
523
583
 
524
584
  kafka-crab-js provides two methods for committing offsets:
525
585
 
526
- | Method | Signature | Description |
527
- | --- | --- | --- |
528
- | `commit` | `commit(topic, partition, offset, mode)` | Traditional commit - you must calculate `offset + 1` |
529
- | `commitMessage` | `commitMessage(message, mode)` | **v2.1.0+**: Simplified commit - automatically handles offset increment |
586
+ | Method | Signature | Description |
587
+ | --------------- | ---------------------------------------- | ----------------------------------------------------------------------- |
588
+ | `commit` | `commit(topic, partition, offset, mode)` | Traditional commit - you must calculate `offset + 1` |
589
+ | `commitMessage` | `commitMessage(message, mode)` | **v2.1.0+**: Simplified commit - automatically handles offset increment |
530
590
 
531
591
  ```javascript
532
592
  // Using commitMessage (recommended for v2.1.0+)
533
- const message = await consumer.recv();
534
- await consumer.commitMessage(message, 'Sync');
593
+ const message = await consumer.recv()
594
+ await consumer.commitMessage(message, 'Sync')
535
595
 
536
596
  // Using commit (traditional)
537
- const message = await consumer.recv();
538
- await consumer.commit(message.topic, message.partition, message.offset + 1, 'Sync');
597
+ const message = await consumer.recv()
598
+ await consumer.commit(message.topic, message.partition, message.offset + 1, 'Sync')
539
599
  ```
540
600
 
541
601
  Both methods support `'Sync'` and `'Async'` commit modes.
542
602
 
543
603
  ### ProducerConfiguration
544
604
 
545
- | Property | Type | Default | Description |
546
- | --- | --- | --- | --- |
547
- | `queueTimeout` | `number` | `5000` | Queue timeout in milliseconds |
548
- | `autoFlush` | `boolean` | `true` | Enable automatic message flushing |
549
- | `configuration` | `Record<string, any>` | `{}` | Additional producer configuration options |
605
+ | Property | Type | Default | Description |
606
+ | --------------- | --------------------- | ------- | ----------------------------------------- |
607
+ | `queueTimeout` | `number` | `5000` | Queue timeout in milliseconds |
608
+ | `autoFlush` | `boolean` | `true` | Enable automatic message flushing |
609
+ | `configuration` | `Record<string, any>` | `{}` | Additional producer configuration options |
550
610
 
551
611
  ### TopicPartitionConfig
552
612
 
553
- | Property | Type | Default | Description |
554
- | --- | --- | --- | --- |
555
- | `topic` | `string` || Topic name |
556
- | `allOffsets` | `OffsetModel` || Offset configuration for all partitions |
557
- | `partitionOffset` | `Array<PartitionOffset>` || Per-partition offset configuration |
558
- | `createTopic` | `boolean` | `false` | **v2.0.0+**: Create topic if it doesn't exist |
559
- | `numPartitions` | `number` | `1` | **v2.0.0+**: Number of partitions when creating topic |
560
- | `replicas` | `number` | `1` | **v2.0.0+**: Number of replicas when creating topic |
613
+ | Property | Type | Default | Description |
614
+ | ----------------- | ------------------------ | ------- | ----------------------------------------------------- |
615
+ | `topic` | `string` | | Topic name |
616
+ | `allOffsets` | `OffsetModel` | | Offset configuration for all partitions |
617
+ | `partitionOffset` | `Array<PartitionOffset>` | | Per-partition offset configuration |
618
+ | `createTopic` | `boolean` | `false` | **v2.0.0+**: Create topic if it doesn't exist |
619
+ | `numPartitions` | `number` | `1` | **v2.0.0+**: Number of partitions when creating topic |
620
+ | `replicas` | `number` | `1` | **v2.0.0+**: Number of replicas when creating topic |
561
621
 
562
622
  You can see the available options here: [librdkafka](https://docs.confluent.io/platform/current/clients/librdkafka/html/md_CONFIGURATION.html).
563
623
 
@@ -601,6 +661,7 @@ For complete configuration options and examples, see the [kafka-crab-js-otel REA
601
661
  #### Examples
602
662
 
603
663
  See comprehensive examples in the `example/` directory:
664
+
604
665
  - `example/otel-tracing-example.mjs` - Complete tracing setup with Jaeger
605
666
  - `example/otel-metrics-example.mjs` - Metrics collection with Prometheus
606
667
  - `example/README.md` - Full documentation for all examples
@@ -612,17 +673,19 @@ See comprehensive examples in the `example/` directory:
612
673
  kafka-crab-js v2.0.0+ includes a comprehensive benchmark suite to compare performance against other popular Kafka clients:
613
674
 
614
675
  ```bash
676
+ cd ../benchmark
677
+
615
678
  # Set up benchmark environment (requires Kafka running locally)
616
- pnpm add -D tsx # For running TypeScript files directly
617
- npx tsx benchmark/utils/setup.ts
679
+ vp install
680
+ vp run setup:consumer
618
681
 
619
682
  # Run consumer performance benchmarks
620
- node benchmark/consumer.ts
683
+ vp run benchmark
621
684
  ```
622
685
 
623
686
  ### Benchmark Results
624
687
 
625
- *Benchmarks run on macOS with Apple M1 chip processing 50,000 messages (December 2024)*
688
+ _Benchmarks run on macOS with Apple M1 chip processing 50,000 messages (December 2024)_
626
689
 
627
690
  ```
628
691
  ╔════════════════════════╤═════════╤══════════════════╤═══════════╤══════════════════════════╗
@@ -640,6 +703,7 @@ node benchmark/consumer.ts
640
703
  ```
641
704
 
642
705
  The benchmark suite compares:
706
+
643
707
  - **kafka-crab-js (serial)**: Single message processing - **43,214 ops/sec**
644
708
  - **kafka-crab-js (batch)**: Batch message processing - **205,985 ops/sec** (fastest)
645
709
  - **node-rdkafka (evented)**: Event-based processing - **24,923 ops/sec**
@@ -647,6 +711,7 @@ The benchmark suite compares:
647
711
  - **kafkajs**: Official KafkaJS client - **834 ops/sec**
648
712
 
649
713
  Performance characteristics:
714
+
650
715
  - **52x faster than kafkajs** in serial mode, **247x faster in batch mode**
651
716
  - **High throughput**: Batch processing provides 4.8x performance improvement over serial mode
652
717
  - **Low latency**: Optimized for both single and batch message processing
@@ -662,41 +727,46 @@ export const topic = 'benchmarks'
662
727
  export const brokers = ['localhost:9092', 'localhost:9093', 'localhost:9094']
663
728
 
664
729
  // Benchmark parameters can be adjusted in consumer.ts:
665
- const iterations = 10_000 // Number of messages to process
666
- const maxBytes = 200 // Maximum message size
730
+ const iterations = 10_000 // Number of messages to process
731
+ const maxBytes = 200 // Maximum message size
667
732
  ```
668
733
 
669
734
  ## Best Practices
670
735
 
671
736
  ### Error Handling
737
+
672
738
  - Always wrap async operations in try-catch blocks
673
739
  - Implement proper error logging and monitoring
674
740
  - Handle both operational and programming errors separately
675
741
 
676
742
  ### Performance
743
+
677
744
  - Use batch operations for high-throughput scenarios
678
745
  - Configure appropriate batch sizes and compression settings
679
746
  - Monitor and tune consumer group performance
680
747
  - Leverage the benchmark suite to optimize your specific use case
681
748
 
682
749
  ### Configuration (v2.0.0+)
750
+
683
751
  - Use the flexible configuration system with proper data types:
684
752
  ```javascript
685
753
  const config = {
686
- 'batch.size': 16384, // number
687
- 'compression.type': 'snappy', // string
688
- 'enable.idempotence': true, // boolean
689
- 'retries': 5 // number
754
+ 'batch.size': 16384, // number
755
+ 'compression.type': 'snappy', // string
756
+ 'enable.idempotence': true, // boolean
757
+ retries: 5, // number
690
758
  }
691
759
  ```
692
760
 
693
761
  ### Message Processing
762
+
694
763
  - Validate message formats before processing
695
764
  - Implement proper serialization/deserialization
696
765
  - Handle message ordering when required
697
766
  - Use topic creation options for better topic management
698
767
 
699
768
  ### Stream Processing (v2.0.0+)
769
+
700
770
  - Configure appropriate `ReadableOptions` for your use case
701
771
  - Use `objectMode: true` for structured message processing
702
772
  - Set appropriate `highWaterMark` based on memory constraints
@@ -714,20 +784,17 @@ const maxBytes = 200 // Maximum message size
714
784
 
715
785
  ```bash
716
786
  # Build the project
717
- pnpm build
787
+ vp run build
718
788
 
719
789
  # Run type checking
720
- pnpm typecheck
721
-
722
- # Run linting
723
- pnpm lint
790
+ vp run typecheck
724
791
 
725
- # Format code
726
- pnpm fmt
792
+ # Run linting and formatting
793
+ vp check
727
794
 
728
795
  # Run benchmarks
729
- npx tsx benchmark/utils/setup.ts
730
- npx tsx benchmark/consumer.ts
796
+ cd ../benchmark
797
+ vp run benchmark
731
798
  ```
732
799
 
733
800
  ## OpenTelemetry Instrumentation
@@ -781,7 +848,7 @@ context.setGlobalContextManager(new AsyncHooksContextManager().enable())
781
848
  enableOtelInstrumentation({
782
849
  captureMessagePayload: true,
783
850
  captureMessageHeaders: true,
784
- producerHook: span => span.setAttribute('messaging.client.kind', 'producer'),
851
+ producerHook: (span) => span.setAttribute('messaging.client.kind', 'producer'),
785
852
  })
786
853
 
787
854
  // 3. Create client with diagnostics enabled
@@ -795,10 +862,12 @@ const client = new KafkaClient({
795
862
  const producer = client.createProducer()
796
863
  await producer.send({
797
864
  topic: 'orders',
798
- messages: [{
799
- payload: Buffer.from(JSON.stringify({ orderId: '123' })),
800
- headers: { 'custom-header': 'foo' },
801
- }],
865
+ messages: [
866
+ {
867
+ payload: Buffer.from(JSON.stringify({ orderId: '123' })),
868
+ headers: { 'custom-header': 'foo' },
869
+ },
870
+ ],
802
871
  })
803
872
 
804
873
  // 5. Use stream consumer with proper cleanup
@@ -809,7 +878,7 @@ const consumer = client.createStreamConsumer({
809
878
 
810
879
  await consumer.subscribe('orders')
811
880
 
812
- consumer.on('data', message => {
881
+ consumer.on('data', (message) => {
813
882
  try {
814
883
  console.log(message.headers?.['custom-header']?.toString())
815
884
  } finally {