kafka-ts 0.0.1-beta → 0.0.3-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/.github/workflows/release.yml +17 -0
  2. package/.prettierrc +3 -2
  3. package/LICENSE +1 -1
  4. package/README.md +48 -35
  5. package/docker-compose.yml +102 -102
  6. package/examples/package-lock.json +28 -27
  7. package/examples/package.json +12 -12
  8. package/examples/src/client.ts +6 -6
  9. package/examples/src/consumer.ts +9 -8
  10. package/examples/src/create-topic.ts +23 -16
  11. package/examples/src/producer.ts +7 -7
  12. package/examples/src/replicator.ts +4 -4
  13. package/examples/src/utils/delay.ts +1 -0
  14. package/examples/src/utils/json.ts +1 -1
  15. package/examples/tsconfig.json +2 -2
  16. package/package.json +21 -15
  17. package/src/__snapshots__/request-handler.test.ts.snap +9 -718
  18. package/src/api/api-versions.ts +2 -2
  19. package/src/api/create-topics.ts +2 -2
  20. package/src/api/delete-topics.ts +2 -2
  21. package/src/api/fetch.ts +3 -3
  22. package/src/api/find-coordinator.ts +2 -2
  23. package/src/api/heartbeat.ts +2 -2
  24. package/src/api/index.ts +18 -18
  25. package/src/api/init-producer-id.ts +2 -2
  26. package/src/api/join-group.ts +3 -3
  27. package/src/api/leave-group.ts +2 -2
  28. package/src/api/list-offsets.ts +3 -3
  29. package/src/api/metadata.ts +3 -3
  30. package/src/api/offset-commit.ts +2 -2
  31. package/src/api/offset-fetch.ts +2 -2
  32. package/src/api/produce.ts +3 -3
  33. package/src/api/sasl-authenticate.ts +2 -2
  34. package/src/api/sasl-handshake.ts +2 -2
  35. package/src/api/sync-group.ts +2 -2
  36. package/src/broker.ts +9 -9
  37. package/src/client.ts +6 -6
  38. package/src/{request-handler.test.ts → cluster.test.ts} +72 -69
  39. package/src/cluster.ts +7 -7
  40. package/src/connection.ts +17 -15
  41. package/src/consumer/consumer-group.ts +14 -14
  42. package/src/consumer/consumer-metadata.ts +2 -2
  43. package/src/consumer/consumer.ts +84 -82
  44. package/src/consumer/fetch-manager.ts +179 -0
  45. package/src/consumer/fetcher.ts +57 -0
  46. package/src/consumer/offset-manager.ts +6 -6
  47. package/src/consumer/processor.ts +47 -0
  48. package/src/distributors/assignments-to-replicas.test.ts +7 -7
  49. package/src/distributors/assignments-to-replicas.ts +1 -1
  50. package/src/distributors/messages-to-topic-partition-leaders.test.ts +6 -6
  51. package/src/index.ts +4 -3
  52. package/src/metadata.ts +4 -4
  53. package/src/producer/producer.ts +8 -8
  54. package/src/types.ts +2 -0
  55. package/src/utils/api.ts +4 -4
  56. package/src/utils/debug.ts +2 -2
  57. package/src/utils/decoder.ts +4 -4
  58. package/src/utils/encoder.ts +6 -6
  59. package/src/utils/error.ts +3 -3
  60. package/src/utils/retrier.ts +1 -1
  61. package/src/utils/tracer.ts +7 -4
  62. package/tsconfig.json +16 -16
  63. package/dist/api/api-versions.d.ts +0 -9
  64. package/dist/api/api-versions.js +0 -24
  65. package/dist/api/create-topics.d.ts +0 -38
  66. package/dist/api/create-topics.js +0 -53
  67. package/dist/api/delete-topics.d.ts +0 -18
  68. package/dist/api/delete-topics.js +0 -33
  69. package/dist/api/fetch.d.ts +0 -77
  70. package/dist/api/fetch.js +0 -106
  71. package/dist/api/find-coordinator.d.ts +0 -21
  72. package/dist/api/find-coordinator.js +0 -39
  73. package/dist/api/heartbeat.d.ts +0 -11
  74. package/dist/api/heartbeat.js +0 -27
  75. package/dist/api/index.d.ts +0 -573
  76. package/dist/api/index.js +0 -164
  77. package/dist/api/init-producer-id.d.ts +0 -13
  78. package/dist/api/init-producer-id.js +0 -29
  79. package/dist/api/join-group.d.ts +0 -34
  80. package/dist/api/join-group.js +0 -51
  81. package/dist/api/leave-group.d.ts +0 -19
  82. package/dist/api/leave-group.js +0 -39
  83. package/dist/api/list-offsets.d.ts +0 -29
  84. package/dist/api/list-offsets.js +0 -48
  85. package/dist/api/metadata.d.ts +0 -40
  86. package/dist/api/metadata.js +0 -58
  87. package/dist/api/offset-commit.d.ts +0 -28
  88. package/dist/api/offset-commit.js +0 -48
  89. package/dist/api/offset-fetch.d.ts +0 -33
  90. package/dist/api/offset-fetch.js +0 -57
  91. package/dist/api/produce.d.ts +0 -53
  92. package/dist/api/produce.js +0 -129
  93. package/dist/api/sasl-authenticate.d.ts +0 -11
  94. package/dist/api/sasl-authenticate.js +0 -23
  95. package/dist/api/sasl-handshake.d.ts +0 -6
  96. package/dist/api/sasl-handshake.js +0 -19
  97. package/dist/api/sync-group.d.ts +0 -24
  98. package/dist/api/sync-group.js +0 -36
  99. package/dist/broker.d.ts +0 -29
  100. package/dist/broker.js +0 -60
  101. package/dist/client.d.ts +0 -23
  102. package/dist/client.js +0 -36
  103. package/dist/cluster.d.ts +0 -24
  104. package/dist/cluster.js +0 -72
  105. package/dist/connection.d.ts +0 -25
  106. package/dist/connection.js +0 -155
  107. package/dist/consumer/consumer-group.d.ts +0 -36
  108. package/dist/consumer/consumer-group.js +0 -182
  109. package/dist/consumer/consumer-metadata.d.ts +0 -7
  110. package/dist/consumer/consumer-metadata.js +0 -14
  111. package/dist/consumer/consumer.d.ts +0 -37
  112. package/dist/consumer/consumer.js +0 -178
  113. package/dist/consumer/metadata.d.ts +0 -24
  114. package/dist/consumer/metadata.js +0 -64
  115. package/dist/consumer/offset-manager.d.ts +0 -22
  116. package/dist/consumer/offset-manager.js +0 -56
  117. package/dist/distributors/assignments-to-replicas.d.ts +0 -17
  118. package/dist/distributors/assignments-to-replicas.js +0 -60
  119. package/dist/distributors/assignments-to-replicas.test.d.ts +0 -1
  120. package/dist/distributors/assignments-to-replicas.test.js +0 -40
  121. package/dist/distributors/messages-to-topic-partition-leaders.d.ts +0 -17
  122. package/dist/distributors/messages-to-topic-partition-leaders.js +0 -15
  123. package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts +0 -1
  124. package/dist/distributors/messages-to-topic-partition-leaders.test.js +0 -30
  125. package/dist/examples/src/replicator.js +0 -34
  126. package/dist/examples/src/utils/json.js +0 -5
  127. package/dist/index.d.ts +0 -3
  128. package/dist/index.js +0 -19
  129. package/dist/metadata.d.ts +0 -24
  130. package/dist/metadata.js +0 -89
  131. package/dist/producer/producer.d.ts +0 -19
  132. package/dist/producer/producer.js +0 -111
  133. package/dist/request-handler.d.ts +0 -16
  134. package/dist/request-handler.js +0 -67
  135. package/dist/request-handler.test.d.ts +0 -1
  136. package/dist/request-handler.test.js +0 -340
  137. package/dist/src/api/api-versions.js +0 -18
  138. package/dist/src/api/create-topics.js +0 -46
  139. package/dist/src/api/delete-topics.js +0 -26
  140. package/dist/src/api/fetch.js +0 -95
  141. package/dist/src/api/find-coordinator.js +0 -34
  142. package/dist/src/api/heartbeat.js +0 -22
  143. package/dist/src/api/index.js +0 -38
  144. package/dist/src/api/init-producer-id.js +0 -24
  145. package/dist/src/api/join-group.js +0 -48
  146. package/dist/src/api/leave-group.js +0 -30
  147. package/dist/src/api/list-offsets.js +0 -39
  148. package/dist/src/api/metadata.js +0 -47
  149. package/dist/src/api/offset-commit.js +0 -39
  150. package/dist/src/api/offset-fetch.js +0 -44
  151. package/dist/src/api/produce.js +0 -119
  152. package/dist/src/api/sync-group.js +0 -31
  153. package/dist/src/broker.js +0 -35
  154. package/dist/src/connection.js +0 -21
  155. package/dist/src/consumer/consumer-group.js +0 -131
  156. package/dist/src/consumer/consumer.js +0 -103
  157. package/dist/src/consumer/metadata.js +0 -52
  158. package/dist/src/consumer/offset-manager.js +0 -23
  159. package/dist/src/index.js +0 -19
  160. package/dist/src/producer/producer.js +0 -84
  161. package/dist/src/request-handler.js +0 -57
  162. package/dist/src/request-handler.test.js +0 -321
  163. package/dist/src/types.js +0 -2
  164. package/dist/src/utils/api.js +0 -5
  165. package/dist/src/utils/decoder.js +0 -161
  166. package/dist/src/utils/encoder.js +0 -137
  167. package/dist/src/utils/error.js +0 -10
  168. package/dist/types.d.ts +0 -9
  169. package/dist/types.js +0 -2
  170. package/dist/utils/api.d.ts +0 -9
  171. package/dist/utils/api.js +0 -5
  172. package/dist/utils/debug.d.ts +0 -2
  173. package/dist/utils/debug.js +0 -11
  174. package/dist/utils/decoder.d.ts +0 -29
  175. package/dist/utils/decoder.js +0 -147
  176. package/dist/utils/delay.d.ts +0 -1
  177. package/dist/utils/delay.js +0 -5
  178. package/dist/utils/encoder.d.ts +0 -28
  179. package/dist/utils/encoder.js +0 -122
  180. package/dist/utils/error.d.ts +0 -11
  181. package/dist/utils/error.js +0 -27
  182. package/dist/utils/memo.d.ts +0 -1
  183. package/dist/utils/memo.js +0 -16
  184. package/dist/utils/retrier.d.ts +0 -10
  185. package/dist/utils/retrier.js +0 -22
  186. package/dist/utils/tracer.d.ts +0 -1
  187. package/dist/utils/tracer.js +0 -26
  188. package/examples/node_modules/.package-lock.json +0 -22
@@ -1,57 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.OFFSET_FETCH = void 0;
4
- const api_1 = require("../utils/api");
5
- const error_1 = require("../utils/error");
6
- exports.OFFSET_FETCH = (0, api_1.createApi)({
7
- apiKey: 9,
8
- apiVersion: 9,
9
- request: (encoder, data) => encoder
10
- .writeUVarInt(0)
11
- .writeCompactArray(data.groups, (encoder, group) => encoder
12
- .writeCompactString(group.groupId)
13
- .writeCompactString(group.memberId)
14
- .writeInt32(group.memberEpoch)
15
- .writeCompactArray(group.topics, (encoder, topic) => encoder
16
- .writeCompactString(topic.name)
17
- .writeCompactArray(topic.partitionIndexes, (encoder, partitionIndex) => encoder.writeInt32(partitionIndex))
18
- .writeUVarInt(0))
19
- .writeUVarInt(0))
20
- .writeBoolean(data.requireStable)
21
- .writeUVarInt(0),
22
- response: (decoder) => {
23
- const result = {
24
- _tag: decoder.readTagBuffer(),
25
- throttleTimeMs: decoder.readInt32(),
26
- groups: decoder.readCompactArray((decoder) => ({
27
- groupId: decoder.readCompactString(),
28
- topics: decoder.readCompactArray((decoder) => ({
29
- name: decoder.readCompactString(),
30
- partitions: decoder.readCompactArray((decoder) => ({
31
- partitionIndex: decoder.readInt32(),
32
- committedOffset: decoder.readInt64(),
33
- committedLeaderEpoch: decoder.readInt32(),
34
- committedMetadata: decoder.readCompactString(),
35
- errorCode: decoder.readInt16(),
36
- _tag: decoder.readTagBuffer(),
37
- })),
38
- _tag: decoder.readTagBuffer(),
39
- })),
40
- errorCode: decoder.readInt16(),
41
- _tag: decoder.readTagBuffer(),
42
- })),
43
- _tag2: decoder.readTagBuffer(),
44
- };
45
- result.groups.forEach((group) => {
46
- if (group.errorCode)
47
- throw new error_1.KafkaTSApiError(group.errorCode, null, result);
48
- group.topics.forEach((topic) => {
49
- topic.partitions.forEach((partition) => {
50
- if (partition.errorCode)
51
- throw new error_1.KafkaTSApiError(partition.errorCode, null, result);
52
- });
53
- });
54
- });
55
- return result;
56
- },
57
- });
@@ -1,53 +0,0 @@
1
- export declare const PRODUCE: import("../utils/api.js").Api<{
2
- transactionalId: string | null;
3
- acks: number;
4
- timeoutMs: number;
5
- topicData: {
6
- name: string;
7
- partitionData: {
8
- index: number;
9
- baseOffset: bigint;
10
- partitionLeaderEpoch: number;
11
- attributes: number;
12
- lastOffsetDelta: number;
13
- baseTimestamp: bigint;
14
- maxTimestamp: bigint;
15
- producerId: bigint;
16
- producerEpoch: number;
17
- baseSequence: number;
18
- records: {
19
- attributes: number;
20
- timestampDelta: bigint;
21
- offsetDelta: number;
22
- key: string | null;
23
- value: string | null;
24
- headers: {
25
- key: string;
26
- value: string;
27
- }[];
28
- }[];
29
- }[];
30
- }[];
31
- }, {
32
- _tag: void;
33
- responses: {
34
- name: string | null;
35
- partitionResponses: {
36
- index: number;
37
- errorCode: number;
38
- baseOffset: bigint;
39
- logAppendTime: bigint;
40
- logStartOffset: bigint;
41
- recordErrors: {
42
- batchIndex: number;
43
- batchIndexError: number;
44
- _tag: void;
45
- }[];
46
- errorMessage: string | null;
47
- _tag: void;
48
- }[];
49
- _tag: void;
50
- }[];
51
- throttleTimeMs: number;
52
- _tag2: void;
53
- }>;
@@ -1,129 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.PRODUCE = void 0;
4
- const api_js_1 = require("../utils/api.js");
5
- const encoder_js_1 = require("../utils/encoder.js");
6
- const error_js_1 = require("../utils/error.js");
7
- exports.PRODUCE = (0, api_js_1.createApi)({
8
- apiKey: 0,
9
- apiVersion: 10,
10
- request: (encoder, data) => encoder
11
- .writeUVarInt(0)
12
- .writeCompactString(data.transactionalId)
13
- .writeInt16(data.acks)
14
- .writeInt32(data.timeoutMs)
15
- .writeCompactArray(data.topicData, (encoder, topic) => encoder
16
- .writeCompactString(topic.name)
17
- .writeCompactArray(topic.partitionData, (encoder, partition) => {
18
- const batchBody = new encoder_js_1.Encoder()
19
- .writeInt16(partition.attributes)
20
- .writeInt32(partition.lastOffsetDelta)
21
- .writeInt64(partition.baseTimestamp)
22
- .writeInt64(partition.maxTimestamp)
23
- .writeInt64(partition.producerId)
24
- .writeInt16(partition.producerEpoch)
25
- .writeInt32(partition.baseSequence)
26
- .writeArray(partition.records, (encoder, record) => {
27
- const recordBody = new encoder_js_1.Encoder()
28
- .writeInt8(record.attributes)
29
- .writeVarLong(record.timestampDelta)
30
- .writeVarInt(record.offsetDelta)
31
- .writeVarIntString(record.key)
32
- .writeVarIntString(record.value)
33
- .writeVarIntArray(record.headers, (encoder, header) => encoder.writeVarIntString(header.key).writeVarIntString(header.value))
34
- .value();
35
- return encoder.writeVarInt(recordBody.length).write(recordBody);
36
- })
37
- .value();
38
- const batchHeader = new encoder_js_1.Encoder()
39
- .writeInt32(partition.partitionLeaderEpoch)
40
- .writeInt8(2) // magic byte
41
- .writeUInt32(unsigned(crc32C(batchBody)))
42
- .write(batchBody)
43
- .value();
44
- const batch = new encoder_js_1.Encoder()
45
- .writeInt64(partition.baseOffset)
46
- .writeInt32(batchHeader.length)
47
- .write(batchHeader)
48
- .value();
49
- return encoder
50
- .writeInt32(partition.index)
51
- .writeUVarInt(batch.length + 1) // batch size
52
- .write(batch)
53
- .writeUVarInt(0);
54
- })
55
- .writeUVarInt(0))
56
- .writeUVarInt(0),
57
- response: (decoder) => {
58
- const result = {
59
- _tag: decoder.readTagBuffer(),
60
- responses: decoder.readCompactArray((response) => ({
61
- name: response.readCompactString(),
62
- partitionResponses: response.readCompactArray((partitionResponse) => ({
63
- index: partitionResponse.readInt32(),
64
- errorCode: partitionResponse.readInt16(),
65
- baseOffset: partitionResponse.readInt64(),
66
- logAppendTime: partitionResponse.readInt64(),
67
- logStartOffset: partitionResponse.readInt64(),
68
- recordErrors: partitionResponse.readCompactArray((recordError) => ({
69
- batchIndex: recordError.readInt32(),
70
- batchIndexError: recordError.readInt16(),
71
- _tag: recordError.readTagBuffer(),
72
- })),
73
- errorMessage: partitionResponse.readCompactString(),
74
- _tag: partitionResponse.readTagBuffer(),
75
- })),
76
- _tag: response.readTagBuffer(),
77
- })),
78
- throttleTimeMs: decoder.readInt32(),
79
- _tag2: decoder.readTagBuffer(),
80
- };
81
- result.responses.forEach((topic) => {
82
- topic.partitionResponses.forEach((partition) => {
83
- if (partition.errorCode !== 0) {
84
- throw new error_js_1.KafkaTSApiError(partition.errorCode, partition.errorMessage, result);
85
- }
86
- });
87
- });
88
- return result;
89
- },
90
- });
91
- const unsigned = (value) => Uint32Array.from([value])[0];
92
- const crc32C = (buffer) => {
93
- let crc = 0 ^ -1;
94
- for (let i = 0; i < buffer.length; i++) {
95
- crc = T[(crc ^ buffer[i]) & 0xff] ^ (crc >>> 8);
96
- }
97
- return (crc ^ -1) >>> 0;
98
- };
99
- const T = new Int32Array([
100
- 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf,
101
- 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c,
102
- 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57,
103
- 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a,
104
- 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e,
105
- 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad,
106
- 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696,
107
- 0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,
108
- 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3,
109
- 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f,
110
- 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, 0x61c69362, 0x93ad1061, 0x80fde395,
111
- 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859,
112
- 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312,
113
- 0x44694011, 0x5739b3e5, 0xa55230e6, 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de,
114
- 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90,
115
- 0x563c5f93, 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c,
116
- 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8,
117
- 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5,
118
- 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e,
119
- 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d,
120
- 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19,
121
- 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8,
122
- 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3,
123
- 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540,
124
- 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a,
125
- 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6,
126
- 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x0105ec76, 0x12551f82,
127
- 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
128
- 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,
129
- ]);
@@ -1,11 +0,0 @@
1
- /// <reference types="node" />
2
- export declare const SASL_AUTHENTICATE: import("../utils/api").Api<{
3
- authBytes: Buffer;
4
- }, {
5
- _tag: void;
6
- errorCode: number;
7
- errorMessage: string | null;
8
- authBytes: Buffer | null;
9
- sessionLifetimeMs: bigint;
10
- _tag2: void;
11
- }>;
@@ -1,23 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SASL_AUTHENTICATE = void 0;
4
- const api_1 = require("../utils/api");
5
- const error_1 = require("../utils/error");
6
- exports.SASL_AUTHENTICATE = (0, api_1.createApi)({
7
- apiKey: 36,
8
- apiVersion: 2,
9
- request: (encoder, data) => encoder.writeUVarInt(0).writeCompactBytes(data.authBytes).writeUVarInt(0),
10
- response: (decoder) => {
11
- const result = {
12
- _tag: decoder.readTagBuffer(),
13
- errorCode: decoder.readInt16(),
14
- errorMessage: decoder.readCompactString(),
15
- authBytes: decoder.readCompactBytes(),
16
- sessionLifetimeMs: decoder.readInt64(),
17
- _tag2: decoder.readTagBuffer(),
18
- };
19
- if (result.errorCode)
20
- throw new error_1.KafkaTSApiError(result.errorCode, result.errorMessage, result);
21
- return result;
22
- },
23
- });
@@ -1,6 +0,0 @@
1
- export declare const SASL_HANDSHAKE: import("../utils/api").Api<{
2
- mechanism: string;
3
- }, {
4
- errorCode: number;
5
- mechanisms: (string | null)[];
6
- }>;
@@ -1,19 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SASL_HANDSHAKE = void 0;
4
- const api_1 = require("../utils/api");
5
- const error_1 = require("../utils/error");
6
- exports.SASL_HANDSHAKE = (0, api_1.createApi)({
7
- apiKey: 17,
8
- apiVersion: 1,
9
- request: (encoder, data) => encoder.writeString(data.mechanism),
10
- response: (decoder) => {
11
- const result = {
12
- errorCode: decoder.readInt16(),
13
- mechanisms: decoder.readArray((mechanism) => mechanism.readString()),
14
- };
15
- if (result.errorCode)
16
- throw new error_1.KafkaTSApiError(result.errorCode, null, result);
17
- return result;
18
- },
19
- });
@@ -1,24 +0,0 @@
1
- export type Assignment = {
2
- [topic: string]: number[];
3
- };
4
- export type MemberAssignment = {
5
- memberId: string;
6
- assignment: Assignment;
7
- };
8
- export declare const SYNC_GROUP: import("../utils/api").Api<{
9
- groupId: string;
10
- generationId: number;
11
- memberId: string;
12
- groupInstanceId: string | null;
13
- protocolType: string | null;
14
- protocolName: string | null;
15
- assignments: MemberAssignment[];
16
- }, {
17
- _tag: void;
18
- throttleTimeMs: number;
19
- errorCode: number;
20
- protocolType: string | null;
21
- protocolName: string | null;
22
- assignments: string;
23
- _tag2: void;
24
- }>;
@@ -1,36 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SYNC_GROUP = void 0;
4
- const api_1 = require("../utils/api");
5
- const error_1 = require("../utils/error");
6
- exports.SYNC_GROUP = (0, api_1.createApi)({
7
- apiKey: 14,
8
- apiVersion: 5,
9
- request: (encoder, data) => encoder
10
- .writeUVarInt(0)
11
- .writeCompactString(data.groupId)
12
- .writeInt32(data.generationId)
13
- .writeCompactString(data.memberId)
14
- .writeCompactString(data.groupInstanceId)
15
- .writeCompactString(data.protocolType)
16
- .writeCompactString(data.protocolName)
17
- .writeCompactArray(data.assignments, (encoder, assignment) => encoder
18
- .writeCompactString(assignment.memberId)
19
- .writeCompactString(JSON.stringify(assignment.assignment))
20
- .writeUVarInt(0))
21
- .writeUVarInt(0),
22
- response: (decoder) => {
23
- const result = {
24
- _tag: decoder.readTagBuffer(),
25
- throttleTimeMs: decoder.readInt32(),
26
- errorCode: decoder.readInt16(),
27
- protocolType: decoder.readCompactString(),
28
- protocolName: decoder.readCompactString(),
29
- assignments: decoder.readCompactString(),
30
- _tag2: decoder.readTagBuffer(),
31
- };
32
- if (result.errorCode)
33
- throw new error_1.KafkaTSApiError(result.errorCode, null, result);
34
- return result;
35
- },
36
- });
package/dist/broker.d.ts DELETED
@@ -1,29 +0,0 @@
1
- /// <reference types="node" />
2
- /// <reference types="node" />
3
- import { TcpSocketConnectOpts } from "net";
4
- import { TLSSocketOptions } from "tls";
5
- import { SendRequest } from "./connection";
6
- export type SASLOptions = {
7
- mechanism: "PLAIN";
8
- username: string;
9
- password: string;
10
- };
11
- type BrokerOptions = {
12
- clientId: string | null;
13
- options: TcpSocketConnectOpts;
14
- sasl: SASLOptions | null;
15
- ssl: TLSSocketOptions | null;
16
- };
17
- export declare class Broker {
18
- private options;
19
- private connection;
20
- sendRequest: SendRequest;
21
- constructor(options: BrokerOptions);
22
- connect(): Promise<this>;
23
- ensureConnected: () => Promise<this>;
24
- disconnect(): Promise<void>;
25
- private validateApiVersions;
26
- private saslHandshake;
27
- private saslAuthenticate;
28
- }
29
- export {};
package/dist/broker.js DELETED
@@ -1,60 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Broker = void 0;
4
- const api_1 = require("./api");
5
- const connection_1 = require("./connection");
6
- const error_1 = require("./utils/error");
7
- const memo_1 = require("./utils/memo");
8
- class Broker {
9
- options;
10
- connection;
11
- sendRequest;
12
- constructor(options) {
13
- this.options = options;
14
- this.connection = new connection_1.Connection({
15
- clientId: this.options.clientId,
16
- connection: this.options.options,
17
- ssl: this.options.ssl,
18
- });
19
- this.sendRequest = this.connection.sendRequest.bind(this.connection);
20
- }
21
- async connect() {
22
- await this.connection.connect();
23
- await this.validateApiVersions();
24
- await this.saslHandshake();
25
- await this.saslAuthenticate();
26
- return this;
27
- }
28
- ensureConnected = (0, memo_1.memo)(() => this.connect());
29
- async disconnect() {
30
- await this.connection.disconnect();
31
- }
32
- async validateApiVersions() {
33
- const { versions } = await this.sendRequest(api_1.API.API_VERSIONS, {});
34
- const apiByKey = Object.fromEntries(Object.values(api_1.API).map((api) => [api.apiKey, api]));
35
- versions.forEach(({ apiKey, minVersion, maxVersion }) => {
36
- if (!apiByKey[apiKey]) {
37
- return;
38
- }
39
- const { apiVersion } = apiByKey[apiKey];
40
- if (apiVersion < minVersion || apiVersion > maxVersion) {
41
- throw new error_1.KafkaTSError(`API ${apiKey} version ${apiVersion} is not supported by the broker`);
42
- }
43
- });
44
- }
45
- async saslHandshake() {
46
- if (!this.options.sasl) {
47
- return;
48
- }
49
- await this.sendRequest(api_1.API.SASL_HANDSHAKE, { mechanism: this.options.sasl.mechanism });
50
- }
51
- async saslAuthenticate() {
52
- if (this.options.sasl?.mechanism !== "PLAIN") {
53
- return;
54
- }
55
- const { username, password } = this.options.sasl;
56
- const authBytes = [null, username, password].join("\u0000");
57
- await this.sendRequest(api_1.API.SASL_AUTHENTICATE, { authBytes: Buffer.from(authBytes) });
58
- }
59
- }
60
- exports.Broker = Broker;
package/dist/client.d.ts DELETED
@@ -1,23 +0,0 @@
1
- /// <reference types="node" />
2
- /// <reference types="node" />
3
- import { TcpSocketConnectOpts } from "net";
4
- import { TLSSocketOptions } from "tls";
5
- import { SASLOptions } from "./broker";
6
- import { Cluster } from "./cluster";
7
- import { Consumer, ConsumerOptions } from "./consumer/consumer";
8
- import { Producer, ProducerOptions } from "./producer/producer";
9
- type ClientOptions = {
10
- clientId?: string | null;
11
- bootstrapServers: TcpSocketConnectOpts[];
12
- sasl?: SASLOptions | null;
13
- ssl?: TLSSocketOptions | null;
14
- };
15
- export declare class Client {
16
- private options;
17
- constructor(options: ClientOptions);
18
- startConsumer(options: ConsumerOptions): Promise<Consumer>;
19
- createProducer(options: ProducerOptions): Producer;
20
- createCluster(): Cluster;
21
- }
22
- export declare const createKafkaClient: (options: ClientOptions) => Client;
23
- export {};
package/dist/client.js DELETED
@@ -1,36 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.createKafkaClient = exports.Client = void 0;
4
- const cluster_1 = require("./cluster");
5
- const consumer_1 = require("./consumer/consumer");
6
- const producer_1 = require("./producer/producer");
7
- class Client {
8
- options;
9
- constructor(options) {
10
- this.options = {
11
- ...options,
12
- clientId: options.clientId ?? null,
13
- sasl: options.sasl ?? null,
14
- ssl: options.ssl ?? null,
15
- };
16
- }
17
- async startConsumer(options) {
18
- const consumer = new consumer_1.Consumer(this.createCluster(), options);
19
- await consumer.start();
20
- return consumer;
21
- }
22
- createProducer(options) {
23
- return new producer_1.Producer(this.createCluster(), options);
24
- }
25
- createCluster() {
26
- return new cluster_1.Cluster({
27
- clientId: this.options.clientId,
28
- bootstrapServers: this.options.bootstrapServers,
29
- sasl: this.options.sasl,
30
- ssl: this.options.ssl,
31
- });
32
- }
33
- }
34
- exports.Client = Client;
35
- const createKafkaClient = (options) => new Client(options);
36
- exports.createKafkaClient = createKafkaClient;
package/dist/cluster.d.ts DELETED
@@ -1,24 +0,0 @@
1
- /// <reference types="node" />
2
- /// <reference types="node" />
3
- import { TcpSocketConnectOpts } from "net";
4
- import { TLSSocketOptions } from "tls";
5
- import { SASLOptions } from "./broker";
6
- import { SendRequest } from "./connection";
7
- type ClusterOptions = {
8
- clientId: string | null;
9
- bootstrapServers: TcpSocketConnectOpts[];
10
- sasl: SASLOptions | null;
11
- ssl: TLSSocketOptions | null;
12
- };
13
- export declare class Cluster {
14
- private options;
15
- private seedBroker;
16
- private brokerById;
17
- constructor(options: ClusterOptions);
18
- connect(): Promise<this>;
19
- disconnect(): Promise<void>;
20
- sendRequest: SendRequest;
21
- sendRequestToNode: (nodeId: number) => SendRequest;
22
- private connectSeedBroker;
23
- }
24
- export {};
package/dist/cluster.js DELETED
@@ -1,72 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Cluster = void 0;
4
- const api_1 = require("./api");
5
- const broker_1 = require("./broker");
6
- const error_1 = require("./utils/error");
7
- class Cluster {
8
- options;
9
- seedBroker;
10
- brokerById = {};
11
- constructor(options) {
12
- this.options = options;
13
- this.seedBroker = new broker_1.Broker({
14
- clientId: this.options.clientId,
15
- sasl: this.options.sasl,
16
- ssl: this.options.ssl,
17
- options: this.options.bootstrapServers[0],
18
- });
19
- }
20
- async connect() {
21
- await this.connectSeedBroker();
22
- const metadata = await this.sendRequest(api_1.API.METADATA, {
23
- allowTopicAutoCreation: false,
24
- includeTopicAuthorizedOperations: false,
25
- topics: [],
26
- });
27
- this.brokerById = Object.fromEntries(metadata.brokers.map(({ nodeId, ...options }) => [
28
- nodeId,
29
- new broker_1.Broker({
30
- clientId: this.options.clientId,
31
- sasl: this.options.sasl,
32
- ssl: this.options.ssl,
33
- options,
34
- }),
35
- ]));
36
- return this;
37
- }
38
- async disconnect() {
39
- await Promise.all([
40
- this.seedBroker.disconnect(),
41
- ...Object.values(this.brokerById).map((broker) => broker.disconnect()),
42
- ]);
43
- }
44
- sendRequest = (...args) => this.seedBroker.sendRequest(...args);
45
- sendRequestToNode = (nodeId) => async (...args) => {
46
- const broker = this.brokerById[nodeId];
47
- if (!broker) {
48
- throw new error_1.ConnectionError(`Broker ${nodeId} is not available`);
49
- }
50
- await broker.ensureConnected();
51
- return broker.sendRequest(...args);
52
- };
53
- async connectSeedBroker() {
54
- const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
55
- for (const options of randomizedBrokers) {
56
- try {
57
- this.seedBroker = await new broker_1.Broker({
58
- clientId: this.options.clientId,
59
- sasl: this.options.sasl,
60
- ssl: this.options.ssl,
61
- options,
62
- }).connect();
63
- return;
64
- }
65
- catch (error) {
66
- console.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
67
- }
68
- }
69
- throw new error_1.KafkaTSError("No seed brokers found");
70
- }
71
- }
72
- exports.Cluster = Cluster;
@@ -1,25 +0,0 @@
1
- /// <reference types="node" />
2
- /// <reference types="node" />
3
- import { TcpSocketConnectOpts } from "net";
4
- import { TLSSocketOptions } from "tls";
5
- import { Api } from "./utils/api";
6
- export type ConnectionOptions = {
7
- clientId: string | null;
8
- connection: TcpSocketConnectOpts;
9
- ssl: TLSSocketOptions | null;
10
- };
11
- export declare class Connection {
12
- private options;
13
- private socket;
14
- private queue;
15
- private lastCorrelationId;
16
- private buffer;
17
- constructor(options: ConnectionOptions);
18
- connect(): Promise<void>;
19
- disconnect(): Promise<void>;
20
- sendRequest<Request, Response>(api: Api<Request, Response>, body: Request): Promise<Response>;
21
- private write;
22
- private handleData;
23
- private nextCorrelationId;
24
- }
25
- export type SendRequest = typeof Connection.prototype.sendRequest;