kafka-ts 0.0.1-beta.2 → 0.0.1-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/README.md +43 -23
  2. package/examples/src/consumer.ts +7 -1
  3. package/examples/src/create-topic.ts +3 -3
  4. package/examples/src/producer.ts +2 -4
  5. package/examples/src/replicator.ts +2 -1
  6. package/package.json +1 -1
  7. package/src/__snapshots__/cluster.test.ts.snap +266 -9
  8. package/src/api/fetch.ts +78 -28
  9. package/src/cluster.test.ts +8 -5
  10. package/src/cluster.ts +42 -37
  11. package/src/codecs/gzip.ts +9 -0
  12. package/src/codecs/index.ts +16 -0
  13. package/src/codecs/none.ts +6 -0
  14. package/src/codecs/types.ts +4 -0
  15. package/src/connection.ts +3 -2
  16. package/src/consumer/consumer-group.ts +8 -6
  17. package/src/consumer/consumer.ts +35 -24
  18. package/src/consumer/fetch-manager.ts +25 -19
  19. package/src/consumer/fetcher.ts +4 -4
  20. package/src/consumer/processor.ts +3 -1
  21. package/src/index.ts +2 -0
  22. package/src/producer/producer.ts +4 -4
  23. package/src/types.ts +2 -2
  24. package/src/utils/api.ts +1 -1
  25. package/src/utils/decoder.ts +13 -3
  26. package/src/utils/logger.ts +37 -0
  27. package/src/utils/mutex.ts +31 -0
  28. package/src/utils/tracer.ts +6 -4
  29. package/src/utils/debug.ts +0 -9
package/README.md CHANGED
@@ -105,32 +105,52 @@ The existing high-level libraries (e.g. kafkajs) are missing a few crucial featu
105
105
 
106
106
  ### `kafka.startConsumer()`
107
107
 
108
- | Name | Type | Required | Default | Description |
109
- | ---------------------- | -------------------------------------- | -------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
110
- | topics | string[] | true | | List of topics to subscribe to |
111
- | groupId | string | false | _null_ | Consumer group id |
112
- | groupInstanceId | string | false | _null_ | Consumer group instance id |
113
- | rackId | string | false | _null_ | Rack id |
114
- | isolationLevel | IsolationLevel | false | | Isolation level |
115
- | sessionTimeoutMs | number | false | 30000 | Session timeout in milliseconds |
116
- | rebalanceTimeoutMs | number | false | 60000 | Rebalance timeout in milliseconds |
117
- | maxWaitMs | number | false | 5000 | Fetch long poll timeout in milliseconds |
118
- | minBytes | number | false | 1 | Minimum number of bytes to wait for before returning a fetch response |
119
- | maxBytes | number | false | 1_048_576 | Maximum number of bytes to return in the fetch response |
120
- | partitionMaxBytes | number | false | 1_048_576 | Maximum number of bytes to return per partition in the fetch response |
121
- | allowTopicAutoCreation | boolean | false | false | Allow kafka to auto-create topic when it doesn't exist |
122
- | fromBeginning | boolean | false | false | Start consuming from the beginning of the topic |
123
- | batchGranularity | BatchGranularity | false | partition | Controls messages split from fetch response. Also controls how often offsets are committed. **onBatch** will include messages:<br/>- **partition** - from a single batch<br/>- **topic** - from all topic partitions<br/>- **broker** - from all assignned topics and partitions |
124
- | concurrency | number | false | 1 | How many batches to process concurrently |
125
- | onMessage | (message: Message) => Promise<unknown> | true | | Callback executed on every message |
126
- | onBatch | (batch: Message[]) => Promise<unknown> | true | | Callback executed on every batch of messages (based on **batchGranuality**) |
108
+ | Name | Type | Required | Default | Description |
109
+ | ---------------------- | -------------------------------------- | -------- | ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
110
+ | topics | string[] | true | | List of topics to subscribe to |
111
+ | groupId | string | false | _null_ | Consumer group id |
112
+ | groupInstanceId | string | false | _null_ | Consumer group instance id |
113
+ | rackId | string | false | _null_ | Rack id |
114
+ | isolationLevel | IsolationLevel | false | IsolationLevel.READ_UNCOMMITTED | Isolation level |
115
+ | sessionTimeoutMs | number | false | 30000 | Session timeout in milliseconds |
116
+ | rebalanceTimeoutMs | number | false | 60000 | Rebalance timeout in milliseconds |
117
+ | maxWaitMs | number | false | 5000 | Fetch long poll timeout in milliseconds |
118
+ | minBytes | number | false | 1 | Minimum number of bytes to wait for before returning a fetch response |
119
+ | maxBytes | number | false | 1_048_576 | Maximum number of bytes to return in the fetch response |
120
+ | partitionMaxBytes | number | false | 1_048_576 | Maximum number of bytes to return per partition in the fetch response |
121
+ | allowTopicAutoCreation | boolean | false | false | Allow kafka to auto-create topic when it doesn't exist |
122
+ | fromBeginning | boolean | false | false | Start consuming from the beginning of the topic |
123
+ | batchGranularity | BatchGranularity | false | partition | Controls messages split from fetch response. Also controls how often offsets are committed. **onBatch** will include messages:<br/>- **partition** - from a single batch<br/>- **topic** - from all topic partitions<br/>- **broker** - from all assignned topics and partitions |
124
+ | concurrency | number | false | 1 | How many batches to process concurrently |
125
+ | onMessage | (message: Message) => Promise<unknown> | true | | Callback executed on every message |
126
+ | onBatch | (batch: Message[]) => Promise<unknown> | true | | Callback executed on every batch of messages (based on **batchGranuality**) |
127
127
 
128
128
  ### `kafka.createProducer()`
129
129
 
130
- | Name | Type | Required | Default | Description |
131
- | ---------------------- | ----------- | -------- | ------- | --------------------------------------------------------------------------------------- |
132
- | allowTopicAutoCreation | boolean | false | false | Allow kafka to auto-create topic when it doesn't exist |
133
- | partitioner | Partitioner | false | | Custom partitioner function. By default, it uses a default java-compatible partitioner. |
130
+ | Name | Type | Required | Default | Description |
131
+ | ---------------------- | ----------- | -------- | ------------------ | --------------------------------------------------------------------------------------- |
132
+ | allowTopicAutoCreation | boolean | false | false | Allow kafka to auto-create topic when it doesn't exist |
133
+ | partitioner | Partitioner | false | defaultPartitioner | Custom partitioner function. By default, it uses a default java-compatible partitioner. |
134
+
135
+ ### `producer.send(messages: Message[])`
136
+
137
+ <!-- export type Message = {
138
+ topic: string;
139
+ partition?: number;
140
+ timestamp?: bigint;
141
+ key?: Buffer | null;
142
+ value: Buffer | null;
143
+ headers?: Record<string, string>;
144
+ }; -->
145
+
146
+ | Name | Type | Required | Default | Description |
147
+ | --------- | ---------------------- | -------- | ------- | -------------------------------------------------------------------------------------------------------------------------- |
148
+ | topic | string | true | | Topic to send the message to |
149
+ | partition | number | false | _null_ | Partition to send the message to. By default partitioned by key. If key is also missing, partition is assigned round-robin |
150
+ | timestamp | bigint | false | _null_ | Message timestamp in milliseconds |
151
+ | key | Buffer \| null | false | _null_ | Message key |
152
+ | value | Buffer \| null | true | | Message value |
153
+ | headers | Record<string, string> | false | _null_ | Message headers |
134
154
 
135
155
  ### Supported SASL mechanisms
136
156
 
package/examples/src/consumer.ts CHANGED
@@ -1,12 +1,18 @@
1
+ import { jsonSerializer, log } from 'kafka-ts';
1
2
  import { kafka } from './client';
3
+ import { delay } from '../../dist/utils/delay';
2
4
 
3
5
  (async () => {
4
6
  const consumer = await kafka.startConsumer({
5
7
  groupId: 'example-group',
6
8
  groupInstanceId: 'example-group-instance',
7
9
  topics: ['my-topic'],
10
+ allowTopicAutoCreation: true,
8
11
  onBatch: (batch) => {
9
- console.log(batch);
12
+ log.info(
13
+ `Received batch: ${JSON.stringify(batch.map((message) => ({ ...message, value: message.value?.toString() })), jsonSerializer)}`,
14
+ );
15
+ log.info(`Latency: ${Date.now() - parseInt(batch[0].timestamp.toString())}ms`)
10
16
  },
11
17
  batchGranularity: 'broker',
12
18
  concurrency: 10,
package/examples/src/create-topic.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { API, API_ERROR, KafkaTSApiError } from 'kafka-ts';
1
+ import { API, API_ERROR, KafkaTSApiError, log } from 'kafka-ts';
2
2
  import { kafka } from './client';
3
3
 
4
4
  (async () => {
@@ -19,7 +19,7 @@ import { kafka } from './client';
19
19
  {
20
20
  name: 'my-topic',
21
21
  numPartitions: 10,
22
- replicationFactor: 3,
22
+ replicationFactor: 1,
23
23
  assignments: [],
24
24
  configs: [],
25
25
  },
@@ -37,7 +37,7 @@ import { kafka } from './client';
37
37
  topics: [{ id: null, name: 'my-topic' }],
38
38
  });
39
39
 
40
- console.log(metadata);
40
+ log.info('Metadata', metadata);
41
41
 
42
42
  await cluster.disconnect();
43
43
  })();
package/examples/src/producer.ts CHANGED
@@ -9,12 +9,10 @@ process.stdout.write('> ');
9
9
  rl.on('line', async (line) => {
10
10
  await producer.send([
11
11
  {
12
- topic: 'example-topic-f',
13
- key: null,
12
+ topic: 'my-topic',
14
13
  value: Buffer.from(line),
15
- partition: 0,
16
14
  },
17
- ]);
15
+ ], { acks: -1 });
18
16
  process.stdout.write('> ');
19
17
  });
20
18
 
package/examples/src/replicator.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { log } from 'kafka-ts';
1
2
  import { kafka } from './client';
2
3
 
3
4
  (async () => {
@@ -15,7 +16,7 @@ import { kafka } from './client';
15
16
  offset: 0n,
16
17
  })),
17
18
  );
18
- console.log(`Replicated ${messages.length} messages`);
19
+ log.info(`Replicated ${messages.length} messages`);
19
20
  },
20
21
  });
21
22
  process.on('SIGINT', async () => {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "kafka-ts",
3
- "version": "0.0.1-beta.2",
3
+ "version": "0.0.1-beta.4",
4
4
  "main": "dist/index.js",
5
5
  "author": "Priit Käärd",
6
6
  "license": "MIT",
package/src/__snapshots__/cluster.test.ts.snap CHANGED
@@ -282,8 +282,8 @@ exports[`Request handler > should create topics 1`] = `
282
282
  "errorCode": 0,
283
283
  "errorMessage": null,
284
284
  "name": "kafka-ts-test-topic",
285
- "numPartitions": 1,
286
- "replicationFactor": 1,
285
+ "numPartitions": 10,
286
+ "replicationFactor": 3,
287
287
  "topicId": "Any<UUID>",
288
288
  },
289
289
  ],
@@ -348,7 +348,11 @@ exports[`Request handler > should fetch messages 1`] = `
348
348
  "baseSequence": 0,
349
349
  "baseTimestamp": 0n,
350
350
  "batchLength": 94,
351
+ "compression": 0,
351
352
  "crc": 0,
353
+ "hasDeleteHorizonMs": false,
354
+ "isControlBatch": false,
355
+ "isTransactional": false,
352
356
  "lastOffsetDelta": 0,
353
357
  "magic": 2,
354
358
  "maxTimestamp": 0n,
@@ -416,6 +420,7 @@ exports[`Request handler > should fetch messages 1`] = `
416
420
  },
417
421
  },
418
422
  ],
423
+ "timestampType": "CreateTime",
419
424
  },
420
425
  ],
421
426
  },
@@ -944,6 +949,132 @@ exports[`Request handler > should request metadata for a topic 1`] = `
944
949
  0,
945
950
  ],
946
951
  },
952
+ {
953
+ "_tag": undefined,
954
+ "errorCode": 0,
955
+ "isrNodes": [
956
+ 0,
957
+ ],
958
+ "leaderEpoch": 0,
959
+ "leaderId": 0,
960
+ "offlineReplicas": [],
961
+ "partitionIndex": 8,
962
+ "replicaNodes": [
963
+ 0,
964
+ ],
965
+ },
966
+ {
967
+ "_tag": undefined,
968
+ "errorCode": 0,
969
+ "isrNodes": [
970
+ 0,
971
+ ],
972
+ "leaderEpoch": 0,
973
+ "leaderId": 0,
974
+ "offlineReplicas": [],
975
+ "partitionIndex": 1,
976
+ "replicaNodes": [
977
+ 0,
978
+ ],
979
+ },
980
+ {
981
+ "_tag": undefined,
982
+ "errorCode": 0,
983
+ "isrNodes": [
984
+ 0,
985
+ ],
986
+ "leaderEpoch": 0,
987
+ "leaderId": 0,
988
+ "offlineReplicas": [],
989
+ "partitionIndex": 2,
990
+ "replicaNodes": [
991
+ 0,
992
+ ],
993
+ },
994
+ {
995
+ "_tag": undefined,
996
+ "errorCode": 0,
997
+ "isrNodes": [
998
+ 0,
999
+ ],
1000
+ "leaderEpoch": 0,
1001
+ "leaderId": 0,
1002
+ "offlineReplicas": [],
1003
+ "partitionIndex": 6,
1004
+ "replicaNodes": [
1005
+ 0,
1006
+ ],
1007
+ },
1008
+ {
1009
+ "_tag": undefined,
1010
+ "errorCode": 0,
1011
+ "isrNodes": [
1012
+ 0,
1013
+ ],
1014
+ "leaderEpoch": 0,
1015
+ "leaderId": 0,
1016
+ "offlineReplicas": [],
1017
+ "partitionIndex": 5,
1018
+ "replicaNodes": [
1019
+ 0,
1020
+ ],
1021
+ },
1022
+ {
1023
+ "_tag": undefined,
1024
+ "errorCode": 0,
1025
+ "isrNodes": [
1026
+ 0,
1027
+ ],
1028
+ "leaderEpoch": 0,
1029
+ "leaderId": 0,
1030
+ "offlineReplicas": [],
1031
+ "partitionIndex": 7,
1032
+ "replicaNodes": [
1033
+ 0,
1034
+ ],
1035
+ },
1036
+ {
1037
+ "_tag": undefined,
1038
+ "errorCode": 0,
1039
+ "isrNodes": [
1040
+ 0,
1041
+ ],
1042
+ "leaderEpoch": 0,
1043
+ "leaderId": 0,
1044
+ "offlineReplicas": [],
1045
+ "partitionIndex": 4,
1046
+ "replicaNodes": [
1047
+ 0,
1048
+ ],
1049
+ },
1050
+ {
1051
+ "_tag": undefined,
1052
+ "errorCode": 0,
1053
+ "isrNodes": [
1054
+ 0,
1055
+ ],
1056
+ "leaderEpoch": 0,
1057
+ "leaderId": 0,
1058
+ "offlineReplicas": [],
1059
+ "partitionIndex": 9,
1060
+ "replicaNodes": [
1061
+ 0,
1062
+ ],
1063
+ },
1064
+ {
1065
+ "_tag": undefined,
1066
+ "errorCode": 0,
1067
+ "isrNodes": [
1068
+ 0,
1069
+ ],
1070
+ "leaderEpoch": 0,
1071
+ "leaderId": 0,
1072
+ "offlineReplicas": [],
1073
+ "partitionIndex": 3,
1074
+ "replicaNodes": [
1075
+ 0,
1076
+ ],
1077
+ },
947
1078
  ],
948
1079
  "topicAuthorizedOperations": -2147483648,
949
1080
  "topicId": "Any<UUID>",
@@ -998,7 +1129,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
998
1129
  "leaderEpoch": 0,
999
1130
  "leaderId": 0,
1000
1131
  "offlineReplicas": [],
1001
- "partitionIndex": 5,
1132
+ "partitionIndex": 9,
1002
1133
  "replicaNodes": [
1003
1134
  0,
1004
1135
  ],
@@ -1026,7 +1157,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
1026
1157
  "leaderEpoch": 0,
1027
1158
  "leaderId": 0,
1028
1159
  "offlineReplicas": [],
1029
- "partitionIndex": 0,
1160
+ "partitionIndex": 1,
1030
1161
  "replicaNodes": [
1031
1162
  0,
1032
1163
  ],
@@ -1054,7 +1185,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
1054
1185
  "leaderEpoch": 0,
1055
1186
  "leaderId": 0,
1056
1187
  "offlineReplicas": [],
1057
- "partitionIndex": 7,
1188
+ "partitionIndex": 5,
1058
1189
  "replicaNodes": [
1059
1190
  0,
1060
1191
  ],
@@ -1068,7 +1199,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
1068
1199
  "leaderEpoch": 0,
1069
1200
  "leaderId": 0,
1070
1201
  "offlineReplicas": [],
1071
- "partitionIndex": 1,
1202
+ "partitionIndex": 3,
1072
1203
  "replicaNodes": [
1073
1204
  0,
1074
1205
  ],
@@ -1096,7 +1227,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
1096
1227
  "leaderEpoch": 0,
1097
1228
  "leaderId": 0,
1098
1229
  "offlineReplicas": [],
1099
- "partitionIndex": 9,
1230
+ "partitionIndex": 0,
1100
1231
  "replicaNodes": [
1101
1232
  0,
1102
1233
  ],
@@ -1110,7 +1241,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
1110
1241
  "leaderEpoch": 0,
1111
1242
  "leaderId": 0,
1112
1243
  "offlineReplicas": [],
1113
- "partitionIndex": 3,
1244
+ "partitionIndex": 4,
1114
1245
  "replicaNodes": [
1115
1246
  0,
1116
1247
  ],
@@ -1124,7 +1255,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
1124
1255
  "leaderEpoch": 0,
1125
1256
  "leaderId": 0,
1126
1257
  "offlineReplicas": [],
1127
- "partitionIndex": 4,
1258
+ "partitionIndex": 7,
1128
1259
  "replicaNodes": [
1129
1260
  0,
1130
1261
  ],
@@ -1153,6 +1284,132 @@ exports[`Request handler > should request metadata for all topics 1`] = `
1153
1284
  0,
1154
1285
  ],
1155
1286
  },
1287
+ {
1288
+ "_tag": undefined,
1289
+ "errorCode": 0,
1290
+ "isrNodes": [
1291
+ 0,
1292
+ ],
1293
+ "leaderEpoch": 0,
1294
+ "leaderId": 0,
1295
+ "offlineReplicas": [],
1296
+ "partitionIndex": 8,
1297
+ "replicaNodes": [
1298
+ 0,
1299
+ ],
1300
+ },
1301
+ {
1302
+ "_tag": undefined,
1303
+ "errorCode": 0,
1304
+ "isrNodes": [
1305
+ 0,
1306
+ ],
1307
+ "leaderEpoch": 0,
1308
+ "leaderId": 0,
1309
+ "offlineReplicas": [],
1310
+ "partitionIndex": 1,
1311
+ "replicaNodes": [
1312
+ 0,
1313
+ ],
1314
+ },
1315
+ {
1316
+ "_tag": undefined,
1317
+ "errorCode": 0,
1318
+ "isrNodes": [
1319
+ 0,
1320
+ ],
1321
+ "leaderEpoch": 0,
1322
+ "leaderId": 0,
1323
+ "offlineReplicas": [],
1324
+ "partitionIndex": 2,
1325
+ "replicaNodes": [
1326
+ 0,
1327
+ ],
1328
+ },
1329
+ {
1330
+ "_tag": undefined,
1331
+ "errorCode": 0,
1332
+ "isrNodes": [
1333
+ 0,
1334
+ ],
1335
+ "leaderEpoch": 0,
1336
+ "leaderId": 0,
1337
+ "offlineReplicas": [],
1338
+ "partitionIndex": 6,
1339
+ "replicaNodes": [
1340
+ 0,
1341
+ ],
1342
+ },
1343
+ {
1344
+ "_tag": undefined,
1345
+ "errorCode": 0,
1346
+ "isrNodes": [
1347
+ 0,
1348
+ ],
1349
+ "leaderEpoch": 0,
1350
+ "leaderId": 0,
1351
+ "offlineReplicas": [],
1352
+ "partitionIndex": 5,
1353
+ "replicaNodes": [
1354
+ 0,
1355
+ ],
1356
+ },
1357
+ {
1358
+ "_tag": undefined,
1359
+ "errorCode": 0,
1360
+ "isrNodes": [
1361
+ 0,
1362
+ ],
1363
+ "leaderEpoch": 0,
1364
+ "leaderId": 0,
1365
+ "offlineReplicas": [],
1366
+ "partitionIndex": 7,
1367
+ "replicaNodes": [
1368
+ 0,
1369
+ ],
1370
+ },
1371
+ {
1372
+ "_tag": undefined,
1373
+ "errorCode": 0,
1374
+ "isrNodes": [
1375
+ 0,
1376
+ ],
1377
+ "leaderEpoch": 0,
1378
+ "leaderId": 0,
1379
+ "offlineReplicas": [],
1380
+ "partitionIndex": 4,
1381
+ "replicaNodes": [
1382
+ 0,
1383
+ ],
1384
+ },
1385
+ {
1386
+ "_tag": undefined,
1387
+ "errorCode": 0,
1388
+ "isrNodes": [
1389
+ 0,
1390
+ ],
1391
+ "leaderEpoch": 0,
1392
+ "leaderId": 0,
1393
+ "offlineReplicas": [],
1394
+ "partitionIndex": 9,
1395
+ "replicaNodes": [
1396
+ 0,
1397
+ ],
1398
+ },
1399
+ {
1400
+ "_tag": undefined,
1401
+ "errorCode": 0,
1402
+ "isrNodes": [
1403
+ 0,
1404
+ ],
1405
+ "leaderEpoch": 0,
1406
+ "leaderId": 0,
1407
+ "offlineReplicas": [],
1408
+ "partitionIndex": 3,
1409
+ "replicaNodes": [
1410
+ 0,
1411
+ ],
1412
+ },
1156
1413
  ],
1157
1414
  "topicAuthorizedOperations": -2147483648,
1158
1415
  "topicId": "Any<UUID>",