kafka-ts 1.3.1-beta.1 → 1.3.1-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +7 -7
  2. package/dist/api/alter-configs.d.ts +26 -0
  3. package/dist/api/alter-configs.js +33 -0
  4. package/dist/api/api-versions.d.ts +5 -2
  5. package/dist/api/api-versions.js +13 -0
  6. package/dist/api/create-topics.d.ts +14 -12
  7. package/dist/api/create-topics.js +104 -12
  8. package/dist/api/delete-topics.d.ts +10 -8
  9. package/dist/api/delete-topics.js +61 -7
  10. package/dist/api/fetch.d.ts +15 -12
  11. package/dist/api/fetch.js +131 -13
  12. package/dist/api/find-coordinator.d.ts +9 -7
  13. package/dist/api/find-coordinator.js +63 -5
  14. package/dist/api/heartbeat.d.ts +7 -5
  15. package/dist/api/heartbeat.js +42 -4
  16. package/dist/api/index.d.ts +47 -118
  17. package/dist/api/init-producer-id.d.ts +7 -5
  18. package/dist/api/init-producer-id.js +53 -9
  19. package/dist/api/join-group.d.ts +9 -7
  20. package/dist/api/join-group.js +95 -6
  21. package/dist/api/leave-group.d.ts +8 -6
  22. package/dist/api/leave-group.js +49 -6
  23. package/dist/api/list-offsets.d.ts +9 -7
  24. package/dist/api/list-offsets.js +85 -8
  25. package/dist/api/metadata.d.ts +10 -9
  26. package/dist/api/metadata.js +109 -8
  27. package/dist/api/offset-commit.d.ts +10 -8
  28. package/dist/api/offset-commit.js +88 -8
  29. package/dist/api/offset-fetch.d.ts +11 -9
  30. package/dist/api/offset-fetch.js +94 -9
  31. package/dist/api/produce.d.ts +8 -10
  32. package/dist/api/produce.js +132 -38
  33. package/dist/api/sasl-authenticate.d.ts +8 -6
  34. package/dist/api/sasl-authenticate.js +43 -3
  35. package/dist/api/sasl-handshake.d.ts +7 -4
  36. package/dist/api/sasl-handshake.js +10 -0
  37. package/dist/api/sync-group.d.ts +7 -5
  38. package/dist/api/sync-group.js +62 -5
  39. package/dist/broker.js +6 -5
  40. package/dist/cluster.test.js +17 -14
  41. package/dist/connection.d.ts +11 -1
  42. package/dist/connection.js +27 -2
  43. package/dist/consumer/consumer.js +13 -9
  44. package/dist/consumer/metadata.d.ts +24 -0
  45. package/dist/consumer/metadata.js +64 -0
  46. package/dist/distributors/messages-to-topic-partition-leaders.d.ts +17 -0
  47. package/dist/distributors/messages-to-topic-partition-leaders.js +15 -0
  48. package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts +1 -0
  49. package/dist/distributors/messages-to-topic-partition-leaders.test.js +30 -0
  50. package/dist/examples/src/replicator.js +34 -0
  51. package/dist/examples/src/utils/json.js +5 -0
  52. package/dist/request-handler.d.ts +16 -0
  53. package/dist/request-handler.js +67 -0
  54. package/dist/request-handler.test.d.ts +1 -0
  55. package/dist/request-handler.test.js +340 -0
  56. package/dist/src/api/api-versions.js +18 -0
  57. package/dist/src/api/create-topics.js +46 -0
  58. package/dist/src/api/delete-topics.js +26 -0
  59. package/dist/src/api/fetch.js +95 -0
  60. package/dist/src/api/find-coordinator.js +34 -0
  61. package/dist/src/api/heartbeat.js +22 -0
  62. package/dist/src/api/index.js +38 -0
  63. package/dist/src/api/init-producer-id.js +24 -0
  64. package/dist/src/api/join-group.js +48 -0
  65. package/dist/src/api/leave-group.js +30 -0
  66. package/dist/src/api/list-offsets.js +39 -0
  67. package/dist/src/api/metadata.js +47 -0
  68. package/dist/src/api/offset-commit.js +39 -0
  69. package/dist/src/api/offset-fetch.js +44 -0
  70. package/dist/src/api/produce.js +119 -0
  71. package/dist/src/api/sync-group.js +31 -0
  72. package/dist/src/broker.js +35 -0
  73. package/dist/src/connection.js +21 -0
  74. package/dist/src/consumer/consumer-group.js +131 -0
  75. package/dist/src/consumer/consumer.js +103 -0
  76. package/dist/src/consumer/metadata.js +52 -0
  77. package/dist/src/consumer/offset-manager.js +23 -0
  78. package/dist/src/index.js +19 -0
  79. package/dist/src/producer/producer.js +84 -0
  80. package/dist/src/request-handler.js +57 -0
  81. package/dist/src/request-handler.test.js +321 -0
  82. package/dist/src/types.js +2 -0
  83. package/dist/src/utils/api.js +5 -0
  84. package/dist/src/utils/decoder.js +161 -0
  85. package/dist/src/utils/encoder.js +137 -0
  86. package/dist/src/utils/error.js +10 -0
  87. package/dist/utils/api.d.ts +4 -1
  88. package/dist/utils/cached.d.ts +3 -0
  89. package/dist/utils/cached.js +19 -0
  90. package/dist/utils/debug.d.ts +2 -0
  91. package/dist/utils/debug.js +11 -0
  92. package/dist/utils/decoder.d.ts +2 -2
  93. package/dist/utils/decoder.js +14 -1
  94. package/dist/utils/encoder.d.ts +1 -0
  95. package/dist/utils/encoder.js +14 -0
  96. package/dist/utils/lock.d.ts +8 -0
  97. package/dist/utils/lock.js +44 -0
  98. package/dist/utils/memo.d.ts +1 -0
  99. package/dist/utils/memo.js +16 -0
  100. package/dist/utils/mutex.d.ts +3 -0
  101. package/dist/utils/mutex.js +32 -0
  102. package/package.json +1 -1
@@ -0,0 +1,48 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.JOIN_GROUP = void 0;
4
+ const api_1 = require("../utils/api");
5
+ const encoder_1 = require("../utils/encoder");
6
+ exports.JOIN_GROUP = (0, api_1.createApi)({
7
+ apiKey: 11,
8
+ apiVersion: 5,
9
+ request: (encoder, data) => encoder
10
+ // .writeUVarInt(0)
11
+ .writeString(data.groupId)
12
+ .writeInt32(data.sessionTimeoutMs)
13
+ .writeInt32(data.rebalanceTimeoutMs)
14
+ .writeString(data.memberId)
15
+ .writeString(data.groupInstanceId)
16
+ .writeString(data.protocolType)
17
+ .writeArray(data.protocols, (encoder, protocol) => {
18
+ const metadata = (0, encoder_1.createEncoder)()
19
+ .writeInt16(protocol.metadata.version)
20
+ .writeArray(protocol.metadata.topics, (encoder, topic) => encoder.writeString(topic))
21
+ .writeBytes(Buffer.alloc(0))
22
+ .value();
23
+ return encoder.writeString(protocol.name).writeBytes(metadata);
24
+ // .writeUVarInt(0)
25
+ ;
26
+ })
27
+ // .writeString(data.reason)
28
+ // .writeUVarInt(0)
29
+ .value(),
30
+ response: (decoder) => ({
31
+ // _tag: decoder.readTagBuffer(),
32
+ throttleTimeMs: decoder.readInt32(),
33
+ errorCode: decoder.readInt16(),
34
+ generationId: decoder.readInt32(),
35
+ // protocolType: decoder.readString(),
36
+ protocolName: decoder.readString(),
37
+ leader: decoder.readString(),
38
+ // skipAssignment: decoder.readBoolean(),
39
+ memberId: decoder.readString(),
40
+ members: decoder.readArray((decoder) => ({
41
+ memberId: decoder.readString(),
42
+ groupInstanceId: decoder.readString(),
43
+ metadata: decoder.readBytes(),
44
+ // _tag: decoder.readTagBuffer(),
45
+ })),
46
+ // _tag2: decoder.readTagBuffer(),
47
+ }),
48
+ });
@@ -0,0 +1,30 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.LEAVE_GROUP = void 0;
4
+ const api_1 = require("../utils/api");
5
+ exports.LEAVE_GROUP = (0, api_1.createApi)({
6
+ apiKey: 13,
7
+ apiVersion: 5,
8
+ request: (encoder, body) => encoder
9
+ .writeUVarInt(0)
10
+ .writeCompactString(body.groupId)
11
+ .writeCompactArray(body.members, (encoder, member) => encoder
12
+ .writeCompactString(member.memberId)
13
+ .writeCompactString(member.groupInstanceId)
14
+ .writeCompactString(member.reason)
15
+ .writeUVarInt(0))
16
+ .writeUVarInt(0)
17
+ .value(),
18
+ response: (decoder) => ({
19
+ _tag: decoder.readTagBuffer(),
20
+ throttleTimeMs: decoder.readInt32(),
21
+ errorCode: decoder.readInt16(),
22
+ members: decoder.readCompactArray((decoder) => ({
23
+ memberId: decoder.readCompactString(),
24
+ groupInstanceId: decoder.readCompactString(),
25
+ errorCode: decoder.readInt16(),
26
+ _tag: decoder.readTagBuffer(),
27
+ })),
28
+ _tag2: decoder.readTagBuffer(),
29
+ }),
30
+ });
@@ -0,0 +1,39 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.LIST_OFFSETS = void 0;
4
+ const api_1 = require("../utils/api");
5
+ exports.LIST_OFFSETS = (0, api_1.createApi)({
6
+ apiKey: 2,
7
+ apiVersion: 8,
8
+ request: (encoder, data) => encoder
9
+ .writeUVarInt(0)
10
+ .writeInt32(data.replicaId)
11
+ .writeInt8(data.isolationLevel)
12
+ .writeCompactArray(data.topics, (encoder, topic) => encoder
13
+ .writeCompactString(topic.name)
14
+ .writeCompactArray(topic.partitions, (encoder, partition) => encoder
15
+ .writeInt32(partition.partitionIndex)
16
+ .writeInt32(partition.currentLeaderEpoch)
17
+ .writeInt64(partition.timestamp)
18
+ .writeUVarInt(0))
19
+ .writeUVarInt(0))
20
+ .writeUVarInt(0)
21
+ .value(),
22
+ response: (decoder) => ({
23
+ _tag: decoder.readTagBuffer(),
24
+ throttleTimeMs: decoder.readInt32(),
25
+ topics: decoder.readCompactArray((decoder) => ({
26
+ name: decoder.readCompactString(),
27
+ partitions: decoder.readCompactArray((decoder) => ({
28
+ partitionIndex: decoder.readInt32(),
29
+ errorCode: decoder.readInt16(),
30
+ timestamp: decoder.readInt64(),
31
+ offset: decoder.readInt64(),
32
+ leaderEpoch: decoder.readInt32(),
33
+ _tag: decoder.readTagBuffer(),
34
+ })),
35
+ _tag: decoder.readTagBuffer(),
36
+ })),
37
+ _tag2: decoder.readTagBuffer(),
38
+ }),
39
+ });
@@ -0,0 +1,47 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.METADATA = void 0;
4
+ const api_1 = require("../utils/api");
5
+ exports.METADATA = (0, api_1.createApi)({
6
+ apiKey: 3,
7
+ apiVersion: 12,
8
+ request: (encoder, data) => encoder
9
+ .writeUVarInt(0)
10
+ .writeCompactArray(data.topics, (encoder, topic) => encoder.writeUUID(topic.id).writeCompactString(topic.name).writeUVarInt(0))
11
+ .writeBoolean(data.allowTopicAutoCreation)
12
+ .writeBoolean(data.includeTopicAuthorizedOperations)
13
+ .writeUVarInt(0)
14
+ .value(),
15
+ response: (decoder) => ({
16
+ _tag: decoder.readTagBuffer(),
17
+ throttleTimeMs: decoder.readInt32(),
18
+ brokers: decoder.readCompactArray((broker) => ({
19
+ nodeId: broker.readInt32(),
20
+ host: broker.readCompactString(),
21
+ port: broker.readInt32(),
22
+ rack: broker.readCompactString(),
23
+ _tag: broker.readTagBuffer(),
24
+ })),
25
+ clusterId: decoder.readCompactString(),
26
+ controllerId: decoder.readInt32(),
27
+ topics: decoder.readCompactArray((topic) => ({
28
+ errorCode: topic.readInt16(),
29
+ name: topic.readCompactString(),
30
+ topicId: topic.readUUID(),
31
+ isInternal: topic.readBoolean(),
32
+ partitions: topic.readCompactArray((partition) => ({
33
+ errorCode: partition.readInt16(),
34
+ partitionIndex: partition.readInt32(),
35
+ leaderId: partition.readInt32(),
36
+ leaderEpoch: partition.readInt32(),
37
+ replicaNodes: partition.readCompactArray((node) => node.readInt32()),
38
+ isrNodes: partition.readCompactArray((node) => node.readInt32()),
39
+ offlineReplicas: partition.readCompactArray((node) => node.readInt32()),
40
+ _tag: partition.readTagBuffer(),
41
+ })),
42
+ topicAuthorizedOperations: topic.readInt32(),
43
+ _tag: topic.readTagBuffer(),
44
+ })),
45
+ _tag2: decoder.readTagBuffer(),
46
+ }),
47
+ });
@@ -0,0 +1,39 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.OFFSET_COMMIT = void 0;
4
+ const api_1 = require("../utils/api");
5
+ exports.OFFSET_COMMIT = (0, api_1.createApi)({
6
+ apiKey: 8,
7
+ apiVersion: 9,
8
+ request: (encoder, data) => encoder
9
+ .writeUVarInt(0)
10
+ .writeCompactString(data.groupId)
11
+ .writeInt32(data.generationIdOrMemberEpoch)
12
+ .writeCompactString(data.memberId)
13
+ .writeCompactString(data.groupInstanceId)
14
+ .writeCompactArray(data.topics, (encoder, topic) => encoder
15
+ .writeCompactString(topic.name)
16
+ .writeCompactArray(topic.partitions, (encoder, partition) => encoder
17
+ .writeInt32(partition.partitionIndex)
18
+ .writeInt64(partition.committedOffset)
19
+ .writeInt32(partition.committedLeaderEpoch)
20
+ .writeCompactString(partition.committedMetadata)
21
+ .writeUVarInt(0))
22
+ .writeUVarInt(0))
23
+ .writeUVarInt(0)
24
+ .value(),
25
+ response: (decoder) => ({
26
+ _tag: decoder.readTagBuffer(),
27
+ throttleTimeMs: decoder.readInt32(),
28
+ topics: decoder.readCompactArray((decoder) => ({
29
+ name: decoder.readCompactString(),
30
+ partitions: decoder.readCompactArray((decoder) => ({
31
+ partitionIndex: decoder.readInt32(),
32
+ errorCode: decoder.readInt16(),
33
+ _tag: decoder.readTagBuffer(),
34
+ })),
35
+ _tag: decoder.readTagBuffer(),
36
+ })),
37
+ _tag2: decoder.readTagBuffer(),
38
+ }),
39
+ });
@@ -0,0 +1,44 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.OFFSET_FETCH = void 0;
4
+ const api_1 = require("../utils/api");
5
+ exports.OFFSET_FETCH = (0, api_1.createApi)({
6
+ apiKey: 9,
7
+ apiVersion: 9,
8
+ request: (encoder, data) => encoder
9
+ .writeUVarInt(0)
10
+ .writeCompactArray(data.groups, (encoder, group) => encoder
11
+ .writeCompactString(group.groupId)
12
+ .writeCompactString(group.memberId)
13
+ .writeInt32(group.memberEpoch)
14
+ .writeCompactArray(group.topics, (encoder, topic) => encoder
15
+ .writeCompactString(topic.name)
16
+ .writeCompactArray(topic.partitionIndexes, (encoder, partitionIndex) => encoder.writeInt32(partitionIndex))
17
+ .writeUVarInt(0))
18
+ .writeUVarInt(0))
19
+ .writeBoolean(data.requireStable)
20
+ .writeUVarInt(0)
21
+ .value(),
22
+ response: (decoder) => ({
23
+ _tag: decoder.readTagBuffer(),
24
+ throttleTimeMs: decoder.readInt32(),
25
+ groups: decoder.readCompactArray((decoder) => ({
26
+ groupId: decoder.readCompactString(),
27
+ topics: decoder.readCompactArray((decoder) => ({
28
+ name: decoder.readCompactString(),
29
+ partitions: decoder.readCompactArray((decoder) => ({
30
+ partitionIndex: decoder.readInt32(),
31
+ committedOffset: decoder.readInt64(),
32
+ committedLeaderEpoch: decoder.readInt32(),
33
+ committedMetadata: decoder.readCompactString(),
34
+ errorCode: decoder.readInt16(),
35
+ _tag: decoder.readTagBuffer(),
36
+ })),
37
+ _tag: decoder.readTagBuffer(),
38
+ })),
39
+ errorCode: decoder.readInt16(),
40
+ _tag: decoder.readTagBuffer(),
41
+ })),
42
+ _tag2: decoder.readTagBuffer(),
43
+ }),
44
+ });
@@ -0,0 +1,119 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.PRODUCE = void 0;
4
+ const api_js_1 = require("../utils/api.js");
5
+ const encoder_js_1 = require("../utils/encoder.js");
6
+ exports.PRODUCE = (0, api_js_1.createApi)({
7
+ apiKey: 0,
8
+ apiVersion: 10,
9
+ request: (encoder, data) => encoder
10
+ .writeUVarInt(0)
11
+ .writeCompactString(data.transactionalId)
12
+ .writeInt16(data.acks)
13
+ .writeInt32(data.timeoutMs)
14
+ .writeCompactArray(data.topicData, (encoder, topic) => encoder
15
+ .writeCompactString(topic.name)
16
+ .writeCompactArray(topic.partitionData, (encoder, partition) => {
17
+ const batchBody = (0, encoder_js_1.createEncoder)()
18
+ .writeInt16(partition.attributes)
19
+ .writeInt32(partition.lastOffsetDelta)
20
+ .writeInt64(partition.baseTimestamp)
21
+ .writeInt64(partition.maxTimestamp)
22
+ .writeInt64(partition.producerId)
23
+ .writeInt16(partition.producerEpoch)
24
+ .writeInt32(partition.baseSequence)
25
+ .writeArray(partition.records, (encoder, record) => {
26
+ const recordBody = (0, encoder_js_1.createEncoder)()
27
+ .writeInt8(record.attributes)
28
+ .writeVarLong(record.timestampDelta)
29
+ .writeVarInt(record.offsetDelta)
30
+ .writeVarIntString(record.key)
31
+ .writeVarIntString(record.value)
32
+ .writeCompactArray(record.headers, (encoder, header) => encoder.writeVarIntString(header.key).writeVarIntString(header.value))
33
+ .value();
34
+ return encoder.writeVarInt(recordBody.length).write(recordBody);
35
+ })
36
+ .value();
37
+ const batchHeader = (0, encoder_js_1.createEncoder)()
38
+ .writeInt32(partition.partitionLeaderEpoch)
39
+ .writeInt8(2) // magic byte
40
+ .writeUInt32(unsigned(crc32C(batchBody)))
41
+ .write(batchBody)
42
+ .value();
43
+ const batch = (0, encoder_js_1.createEncoder)()
44
+ .writeInt64(partition.baseOffset)
45
+ .writeInt32(batchHeader.length)
46
+ .write(batchHeader)
47
+ .value();
48
+ return encoder
49
+ .writeInt32(partition.index)
50
+ .writeUVarInt(batch.length + 1) // batch size
51
+ .write(batch)
52
+ .writeUVarInt(0);
53
+ })
54
+ .writeUVarInt(0))
55
+ .writeUVarInt(0)
56
+ .value(),
57
+ response: (decoder) => ({
58
+ _tag: decoder.readTagBuffer(),
59
+ responses: decoder.readCompactArray((response) => ({
60
+ name: response.readCompactString(),
61
+ partitionResponses: response.readCompactArray((partitionResponse) => ({
62
+ index: partitionResponse.readInt32(),
63
+ errorCode: partitionResponse.readInt16(),
64
+ baseOffset: partitionResponse.readInt64(),
65
+ logAppendTime: partitionResponse.readInt64(),
66
+ logStartOffset: partitionResponse.readInt64(),
67
+ recordErrors: partitionResponse.readCompactArray((recordError) => ({
68
+ batchIndex: recordError.readInt32(),
69
+ batchIndexError: recordError.readInt16(),
70
+ _tag: recordError.readTagBuffer(),
71
+ })),
72
+ errorMessage: partitionResponse.readCompactString(),
73
+ _tag: partitionResponse.readTagBuffer(),
74
+ })),
75
+ _tag: response.readTagBuffer(),
76
+ })),
77
+ throttleTimeMs: decoder.readInt32(),
78
+ _tag2: decoder.readTagBuffer(),
79
+ }),
80
+ });
81
+ const unsigned = (value) => Uint32Array.from([value])[0];
82
+ const crc32C = (buffer) => {
83
+ let crc = 0 ^ -1;
84
+ for (let i = 0; i < buffer.length; i++) {
85
+ crc = T[(crc ^ buffer[i]) & 0xff] ^ (crc >>> 8);
86
+ }
87
+ return (crc ^ -1) >>> 0;
88
+ };
89
+ const T = new Int32Array([
90
+ 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf,
91
+ 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c,
92
+ 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57,
93
+ 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a,
94
+ 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e,
95
+ 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad,
96
+ 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696,
97
+ 0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,
98
+ 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3,
99
+ 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f,
100
+ 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, 0x61c69362, 0x93ad1061, 0x80fde395,
101
+ 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859,
102
+ 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312,
103
+ 0x44694011, 0x5739b3e5, 0xa55230e6, 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de,
104
+ 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90,
105
+ 0x563c5f93, 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c,
106
+ 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8,
107
+ 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5,
108
+ 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e,
109
+ 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d,
110
+ 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19,
111
+ 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8,
112
+ 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3,
113
+ 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540,
114
+ 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a,
115
+ 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6,
116
+ 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x0105ec76, 0x12551f82,
117
+ 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
118
+ 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,
119
+ ]);
@@ -0,0 +1,31 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.SYNC_GROUP = void 0;
4
+ const api_1 = require("../utils/api");
5
+ exports.SYNC_GROUP = (0, api_1.createApi)({
6
+ apiKey: 14,
7
+ apiVersion: 5,
8
+ request: (encoder, data) => encoder
9
+ .writeUVarInt(0)
10
+ .writeCompactString(data.groupId)
11
+ .writeInt32(data.generationId)
12
+ .writeCompactString(data.memberId)
13
+ .writeCompactString(data.groupInstanceId)
14
+ .writeCompactString(data.protocolType)
15
+ .writeCompactString(data.protocolName)
16
+ .writeCompactArray(data.assignments, (encoder, assignment) => encoder
17
+ .writeCompactString(assignment.memberId)
18
+ .writeCompactString(JSON.stringify(assignment.assignment))
19
+ .writeUVarInt(0))
20
+ .writeUVarInt(0)
21
+ .value(),
22
+ response: (decoder) => ({
23
+ _tag: decoder.readTagBuffer(),
24
+ throttleTimeMs: decoder.readInt32(),
25
+ errorCode: decoder.readInt16(),
26
+ protocolType: decoder.readCompactString(),
27
+ protocolName: decoder.readCompactString(),
28
+ assignments: decoder.readCompactString(),
29
+ _tag2: decoder.readTagBuffer(),
30
+ }),
31
+ });
@@ -0,0 +1,35 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.connectBroker = void 0;
4
+ const api_1 = require("./api");
5
+ const api_versions_1 = require("./api/api-versions");
6
+ const connection_1 = require("./connection");
7
+ const request_handler_1 = require("./request-handler");
8
+ const error_1 = require("./utils/error");
9
+ const connectBroker = async ({ clientId, options }) => {
10
+ const connection = (0, connection_1.createConnection)(options);
11
+ await connection.connect();
12
+ const { sendRequest } = (0, request_handler_1.createRequestHandler)({ clientId, connection });
13
+ await validateApiVersions(sendRequest);
14
+ return {
15
+ clientId,
16
+ sendRequest,
17
+ disconnect: connection.disconnect,
18
+ host: options.host,
19
+ port: options.port,
20
+ };
21
+ };
22
+ exports.connectBroker = connectBroker;
23
+ const validateApiVersions = async (sendRequest) => {
24
+ const { versions } = await sendRequest(api_versions_1.API_VERSIONS, {});
25
+ const apiByKey = Object.fromEntries(Object.values(api_1.API).map((api) => [api.apiKey, api]));
26
+ versions.forEach(({ apiKey, minVersion, maxVersion }) => {
27
+ if (!apiByKey[apiKey]) {
28
+ return;
29
+ }
30
+ const { apiVersion } = apiByKey[apiKey];
31
+ if (apiVersion < minVersion || apiVersion > maxVersion) {
32
+ throw new error_1.KafkaTSError(`API ${apiKey} version ${apiVersion} is not supported by the broker`);
33
+ }
34
+ });
35
+ };
@@ -0,0 +1,21 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createConnection = void 0;
4
+ const net_1 = require("net");
5
+ const createConnection = (options) => {
6
+ const socket = new net_1.Socket();
7
+ const connect = () => {
8
+ return new Promise((resolve, reject) => {
9
+ socket.connect(options);
10
+ socket.once('connect', resolve);
11
+ socket.once('error', reject);
12
+ });
13
+ };
14
+ const disconnect = () => {
15
+ return new Promise((resolve) => {
16
+ socket.end(resolve);
17
+ });
18
+ };
19
+ return { connect, disconnect, socket };
20
+ };
21
+ exports.createConnection = createConnection;
@@ -0,0 +1,131 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createConsumerGroup = void 0;
4
+ const api_1 = require("../api");
5
+ const find_coordinator_1 = require("../api/find-coordinator");
6
+ const broker_1 = require("../broker");
7
+ const createConsumerGroup = ({ topics, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, coordinator, metadata, offsetManager, }) => {
8
+ let memberId = "";
9
+ let generationId = -1;
10
+ let leaderId = "";
11
+ let memberIds = [];
12
+ const findCoordinator = async () => {
13
+ const { coordinators } = await coordinator.sendRequest(api_1.API.FIND_COORDINATOR, {
14
+ keyType: find_coordinator_1.KEY_TYPE.GROUP,
15
+ keys: [groupId],
16
+ });
17
+ if (coordinators[0].host !== coordinator.host || coordinators[0].port !== coordinator.port) {
18
+ await coordinator.disconnect();
19
+ coordinator = await (0, broker_1.connectBroker)({
20
+ clientId: coordinator.clientId,
21
+ options: { host: coordinators[0].host, port: coordinators[0].port },
22
+ });
23
+ }
24
+ };
25
+ const joinGroup = async () => {
26
+ const response = await coordinator.sendRequest(api_1.API.JOIN_GROUP, {
27
+ groupId,
28
+ groupInstanceId,
29
+ memberId,
30
+ sessionTimeoutMs,
31
+ rebalanceTimeoutMs,
32
+ protocolType: "consumer",
33
+ protocols: [{ name: "RoundRobinAssigner", metadata: { version: 0, topics } }],
34
+ reason: null,
35
+ });
36
+ memberId = response.memberId;
37
+ generationId = response.generationId;
38
+ leaderId = response.leader;
39
+ memberIds = response.members.map((member) => member.memberId);
40
+ if (response.errorCode === api_1.API_ERROR.MEMBER_ID_REQUIRED) {
41
+ return joinGroup();
42
+ }
43
+ };
44
+ const syncGroup = async () => {
45
+ let assignments = [];
46
+ if (memberId === leaderId) {
47
+ const memberAssignments = Object.entries(metadata.getTopicPartitions())
48
+ .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
49
+ .reduce((acc, { topic, partition }, index) => {
50
+ const memberId = memberIds[index % memberIds.length];
51
+ acc[memberId] ??= {};
52
+ acc[memberId][topic] ??= [];
53
+ acc[memberId][topic].push(partition);
54
+ return acc;
55
+ }, {});
56
+ assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
57
+ }
58
+ const response = await coordinator.sendRequest(api_1.API.SYNC_GROUP, {
59
+ groupId,
60
+ groupInstanceId,
61
+ memberId,
62
+ generationId,
63
+ protocolType: "consumer",
64
+ protocolName: "RoundRobinAssigner",
65
+ assignments,
66
+ });
67
+ metadata.setAssignment(JSON.parse(response.assignments));
68
+ };
69
+ const offsetFetch = async () => {
70
+ const response = await coordinator.sendRequest(api_1.API.OFFSET_FETCH, {
71
+ groups: [
72
+ {
73
+ groupId,
74
+ memberId,
75
+ memberEpoch: -1,
76
+ topics: topics.map((topic) => ({ name: topic, partitionIndexes: metadata.getAssignment()[topic] })),
77
+ },
78
+ ],
79
+ requireStable: true,
80
+ });
81
+ response.groups.forEach((group) => {
82
+ group.topics.forEach((topic) => {
83
+ topic.partitions
84
+ .filter(({ committedOffset }) => committedOffset >= 0)
85
+ .forEach(({ partitionIndex, committedOffset }) => offsetManager.resolve(topic.name, partitionIndex, committedOffset));
86
+ });
87
+ });
88
+ };
89
+ const offsetCommit = async () => {
90
+ await coordinator.sendRequest(api_1.API.OFFSET_COMMIT, {
91
+ groupId,
92
+ groupInstanceId,
93
+ memberId,
94
+ generationIdOrMemberEpoch: generationId,
95
+ topics: Object.entries(offsetManager.getPendingOffsets()).map(([topic, partitions]) => ({
96
+ name: topic,
97
+ partitions: Object.entries(partitions).map(([partition, offset]) => ({
98
+ partitionIndex: parseInt(partition),
99
+ committedOffset: offset,
100
+ committedLeaderEpoch: -1,
101
+ committedMetadata: null,
102
+ })),
103
+ })),
104
+ });
105
+ offsetManager.flush();
106
+ };
107
+ const leaveGroup = async () => {
108
+ if (!groupId) {
109
+ return;
110
+ }
111
+ await coordinator.sendRequest(api_1.API.LEAVE_GROUP, {
112
+ groupId,
113
+ members: [{ memberId, groupInstanceId, reason: null }],
114
+ });
115
+ };
116
+ return {
117
+ join: async () => {
118
+ await findCoordinator();
119
+ await joinGroup();
120
+ await syncGroup();
121
+ await offsetFetch();
122
+ },
123
+ commit: async () => {
124
+ await offsetCommit();
125
+ },
126
+ leave: async () => {
127
+ await leaveGroup();
128
+ },
129
+ };
130
+ };
131
+ exports.createConsumerGroup = createConsumerGroup;