kafka-ts 0.0.1-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/.prettierrc +7 -0
  2. package/LICENSE +24 -0
  3. package/README.md +88 -0
  4. package/certs/ca.crt +29 -0
  5. package/certs/ca.key +52 -0
  6. package/certs/ca.srl +1 -0
  7. package/certs/kafka.crt +29 -0
  8. package/certs/kafka.csr +26 -0
  9. package/certs/kafka.key +52 -0
  10. package/certs/kafka.keystore.jks +0 -0
  11. package/certs/kafka.truststore.jks +0 -0
  12. package/dist/api/api-versions.d.ts +9 -0
  13. package/dist/api/api-versions.js +24 -0
  14. package/dist/api/create-topics.d.ts +38 -0
  15. package/dist/api/create-topics.js +53 -0
  16. package/dist/api/delete-topics.d.ts +18 -0
  17. package/dist/api/delete-topics.js +33 -0
  18. package/dist/api/fetch.d.ts +77 -0
  19. package/dist/api/fetch.js +106 -0
  20. package/dist/api/find-coordinator.d.ts +21 -0
  21. package/dist/api/find-coordinator.js +39 -0
  22. package/dist/api/heartbeat.d.ts +11 -0
  23. package/dist/api/heartbeat.js +27 -0
  24. package/dist/api/index.d.ts +573 -0
  25. package/dist/api/index.js +164 -0
  26. package/dist/api/init-producer-id.d.ts +13 -0
  27. package/dist/api/init-producer-id.js +29 -0
  28. package/dist/api/join-group.d.ts +34 -0
  29. package/dist/api/join-group.js +51 -0
  30. package/dist/api/leave-group.d.ts +19 -0
  31. package/dist/api/leave-group.js +39 -0
  32. package/dist/api/list-offsets.d.ts +29 -0
  33. package/dist/api/list-offsets.js +48 -0
  34. package/dist/api/metadata.d.ts +40 -0
  35. package/dist/api/metadata.js +58 -0
  36. package/dist/api/offset-commit.d.ts +28 -0
  37. package/dist/api/offset-commit.js +48 -0
  38. package/dist/api/offset-fetch.d.ts +33 -0
  39. package/dist/api/offset-fetch.js +57 -0
  40. package/dist/api/produce.d.ts +53 -0
  41. package/dist/api/produce.js +129 -0
  42. package/dist/api/sasl-authenticate.d.ts +11 -0
  43. package/dist/api/sasl-authenticate.js +23 -0
  44. package/dist/api/sasl-handshake.d.ts +6 -0
  45. package/dist/api/sasl-handshake.js +19 -0
  46. package/dist/api/sync-group.d.ts +24 -0
  47. package/dist/api/sync-group.js +36 -0
  48. package/dist/broker.d.ts +29 -0
  49. package/dist/broker.js +60 -0
  50. package/dist/client.d.ts +23 -0
  51. package/dist/client.js +36 -0
  52. package/dist/cluster.d.ts +24 -0
  53. package/dist/cluster.js +72 -0
  54. package/dist/connection.d.ts +25 -0
  55. package/dist/connection.js +155 -0
  56. package/dist/consumer/consumer-group.d.ts +36 -0
  57. package/dist/consumer/consumer-group.js +182 -0
  58. package/dist/consumer/consumer-metadata.d.ts +7 -0
  59. package/dist/consumer/consumer-metadata.js +14 -0
  60. package/dist/consumer/consumer.d.ts +37 -0
  61. package/dist/consumer/consumer.js +178 -0
  62. package/dist/consumer/metadata.d.ts +24 -0
  63. package/dist/consumer/metadata.js +64 -0
  64. package/dist/consumer/offset-manager.d.ts +22 -0
  65. package/dist/consumer/offset-manager.js +56 -0
  66. package/dist/distributors/assignments-to-replicas.d.ts +17 -0
  67. package/dist/distributors/assignments-to-replicas.js +60 -0
  68. package/dist/distributors/assignments-to-replicas.test.d.ts +1 -0
  69. package/dist/distributors/assignments-to-replicas.test.js +40 -0
  70. package/dist/distributors/messages-to-topic-partition-leaders.d.ts +17 -0
  71. package/dist/distributors/messages-to-topic-partition-leaders.js +15 -0
  72. package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts +1 -0
  73. package/dist/distributors/messages-to-topic-partition-leaders.test.js +30 -0
  74. package/dist/examples/src/replicator.js +34 -0
  75. package/dist/examples/src/utils/json.js +5 -0
  76. package/dist/index.d.ts +3 -0
  77. package/dist/index.js +19 -0
  78. package/dist/metadata.d.ts +24 -0
  79. package/dist/metadata.js +89 -0
  80. package/dist/producer/producer.d.ts +19 -0
  81. package/dist/producer/producer.js +111 -0
  82. package/dist/request-handler.d.ts +16 -0
  83. package/dist/request-handler.js +67 -0
  84. package/dist/request-handler.test.d.ts +1 -0
  85. package/dist/request-handler.test.js +340 -0
  86. package/dist/src/api/api-versions.js +18 -0
  87. package/dist/src/api/create-topics.js +46 -0
  88. package/dist/src/api/delete-topics.js +26 -0
  89. package/dist/src/api/fetch.js +95 -0
  90. package/dist/src/api/find-coordinator.js +34 -0
  91. package/dist/src/api/heartbeat.js +22 -0
  92. package/dist/src/api/index.js +38 -0
  93. package/dist/src/api/init-producer-id.js +24 -0
  94. package/dist/src/api/join-group.js +48 -0
  95. package/dist/src/api/leave-group.js +30 -0
  96. package/dist/src/api/list-offsets.js +39 -0
  97. package/dist/src/api/metadata.js +47 -0
  98. package/dist/src/api/offset-commit.js +39 -0
  99. package/dist/src/api/offset-fetch.js +44 -0
  100. package/dist/src/api/produce.js +119 -0
  101. package/dist/src/api/sync-group.js +31 -0
  102. package/dist/src/broker.js +35 -0
  103. package/dist/src/connection.js +21 -0
  104. package/dist/src/consumer/consumer-group.js +131 -0
  105. package/dist/src/consumer/consumer.js +103 -0
  106. package/dist/src/consumer/metadata.js +52 -0
  107. package/dist/src/consumer/offset-manager.js +23 -0
  108. package/dist/src/index.js +19 -0
  109. package/dist/src/producer/producer.js +84 -0
  110. package/dist/src/request-handler.js +57 -0
  111. package/dist/src/request-handler.test.js +321 -0
  112. package/dist/src/types.js +2 -0
  113. package/dist/src/utils/api.js +5 -0
  114. package/dist/src/utils/decoder.js +161 -0
  115. package/dist/src/utils/encoder.js +137 -0
  116. package/dist/src/utils/error.js +10 -0
  117. package/dist/types.d.ts +9 -0
  118. package/dist/types.js +2 -0
  119. package/dist/utils/api.d.ts +9 -0
  120. package/dist/utils/api.js +5 -0
  121. package/dist/utils/debug.d.ts +2 -0
  122. package/dist/utils/debug.js +11 -0
  123. package/dist/utils/decoder.d.ts +29 -0
  124. package/dist/utils/decoder.js +147 -0
  125. package/dist/utils/delay.d.ts +1 -0
  126. package/dist/utils/delay.js +5 -0
  127. package/dist/utils/encoder.d.ts +28 -0
  128. package/dist/utils/encoder.js +122 -0
  129. package/dist/utils/error.d.ts +11 -0
  130. package/dist/utils/error.js +27 -0
  131. package/dist/utils/memo.d.ts +1 -0
  132. package/dist/utils/memo.js +16 -0
  133. package/dist/utils/retrier.d.ts +10 -0
  134. package/dist/utils/retrier.js +22 -0
  135. package/dist/utils/tracer.d.ts +1 -0
  136. package/dist/utils/tracer.js +26 -0
  137. package/docker-compose.yml +104 -0
  138. package/examples/node_modules/.package-lock.json +22 -0
  139. package/examples/package-lock.json +30 -0
  140. package/examples/package.json +14 -0
  141. package/examples/src/client.ts +9 -0
  142. package/examples/src/consumer.ts +17 -0
  143. package/examples/src/create-topic.ts +37 -0
  144. package/examples/src/producer.ts +24 -0
  145. package/examples/src/replicator.ts +25 -0
  146. package/examples/src/utils/json.ts +1 -0
  147. package/examples/tsconfig.json +7 -0
  148. package/log4j.properties +95 -0
  149. package/package.json +17 -0
  150. package/scripts/generate-certs.sh +24 -0
  151. package/src/__snapshots__/request-handler.test.ts.snap +1687 -0
  152. package/src/api/api-versions.ts +21 -0
  153. package/src/api/create-topics.ts +78 -0
  154. package/src/api/delete-topics.ts +42 -0
  155. package/src/api/fetch.ts +143 -0
  156. package/src/api/find-coordinator.ts +39 -0
  157. package/src/api/heartbeat.ts +33 -0
  158. package/src/api/index.ts +164 -0
  159. package/src/api/init-producer-id.ts +35 -0
  160. package/src/api/join-group.ts +67 -0
  161. package/src/api/leave-group.ts +48 -0
  162. package/src/api/list-offsets.ts +65 -0
  163. package/src/api/metadata.ts +66 -0
  164. package/src/api/offset-commit.ts +67 -0
  165. package/src/api/offset-fetch.ts +74 -0
  166. package/src/api/produce.ts +173 -0
  167. package/src/api/sasl-authenticate.ts +21 -0
  168. package/src/api/sasl-handshake.ts +16 -0
  169. package/src/api/sync-group.ts +54 -0
  170. package/src/broker.ts +74 -0
  171. package/src/client.ts +47 -0
  172. package/src/cluster.ts +87 -0
  173. package/src/connection.ts +141 -0
  174. package/src/consumer/consumer-group.ts +209 -0
  175. package/src/consumer/consumer-metadata.ts +14 -0
  176. package/src/consumer/consumer.ts +229 -0
  177. package/src/consumer/offset-manager.ts +93 -0
  178. package/src/distributors/assignments-to-replicas.test.ts +43 -0
  179. package/src/distributors/assignments-to-replicas.ts +85 -0
  180. package/src/distributors/messages-to-topic-partition-leaders.test.ts +32 -0
  181. package/src/distributors/messages-to-topic-partition-leaders.ts +19 -0
  182. package/src/index.ts +3 -0
  183. package/src/metadata.ts +122 -0
  184. package/src/producer/producer.ts +132 -0
  185. package/src/request-handler.test.ts +366 -0
  186. package/src/types.ts +9 -0
  187. package/src/utils/api.ts +11 -0
  188. package/src/utils/debug.ts +9 -0
  189. package/src/utils/decoder.ts +168 -0
  190. package/src/utils/delay.ts +1 -0
  191. package/src/utils/encoder.ts +141 -0
  192. package/src/utils/error.ts +21 -0
  193. package/src/utils/memo.ts +12 -0
  194. package/src/utils/retrier.ts +39 -0
  195. package/src/utils/tracer.ts +28 -0
  196. package/tsconfig.json +17 -0
@@ -0,0 +1,106 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.FETCH = void 0;
4
+ const api_1 = require("../utils/api");
5
+ const error_1 = require("../utils/error");
6
+ exports.FETCH = (0, api_1.createApi)({
7
+ apiKey: 1,
8
+ apiVersion: 16,
9
+ request: (encoder, data) => encoder
10
+ .writeUVarInt(0)
11
+ .writeInt32(data.maxWaitMs)
12
+ .writeInt32(data.minBytes)
13
+ .writeInt32(data.maxBytes)
14
+ .writeInt8(data.isolationLevel)
15
+ .writeInt32(data.sessionId)
16
+ .writeInt32(data.sessionEpoch)
17
+ .writeCompactArray(data.topics, (encoder, topic) => encoder
18
+ .writeUUID(topic.topicId)
19
+ .writeCompactArray(topic.partitions, (encoder, partition) => encoder
20
+ .writeInt32(partition.partition)
21
+ .writeInt32(partition.currentLeaderEpoch)
22
+ .writeInt64(partition.fetchOffset)
23
+ .writeInt32(partition.lastFetchedEpoch)
24
+ .writeInt64(partition.logStartOffset)
25
+ .writeInt32(partition.partitionMaxBytes)
26
+ .writeUVarInt(0))
27
+ .writeUVarInt(0))
28
+ .writeCompactArray(data.forgottenTopicsData, (encoder, forgottenTopic) => encoder
29
+ .writeUUID(forgottenTopic.topicId)
30
+ .writeCompactArray(forgottenTopic.partitions, (encoder, partition) => encoder.writeInt32(partition))
31
+ .writeUVarInt(0))
32
+ .writeCompactString(data.rackId)
33
+ .writeUVarInt(0),
34
+ response: (decoder) => {
35
+ const result = {
36
+ _tag: decoder.readTagBuffer(),
37
+ throttleTimeMs: decoder.readInt32(),
38
+ errorCode: decoder.readInt16(),
39
+ sessionId: decoder.readInt32(),
40
+ responses: decoder.readCompactArray((response) => ({
41
+ topicId: response.readUUID(),
42
+ partitions: response.readCompactArray((partition) => ({
43
+ partitionIndex: partition.readInt32(),
44
+ errorCode: partition.readInt16(),
45
+ highWatermark: partition.readInt64(),
46
+ lastStableOffset: partition.readInt64(),
47
+ logStartOffset: partition.readInt64(),
48
+ abortedTransactions: partition.readCompactArray((abortedTransaction) => ({
49
+ producerId: abortedTransaction.readInt64(),
50
+ firstOffset: abortedTransaction.readInt64(),
51
+ _tag: abortedTransaction.readTagBuffer(),
52
+ })),
53
+ preferredReadReplica: partition.readInt32(),
54
+ records: decodeRecords(partition),
55
+ _tag: partition.readTagBuffer(),
56
+ })),
57
+ _tag: response.readTagBuffer(),
58
+ })),
59
+ _tag2: decoder.readTagBuffer(),
60
+ };
61
+ if (result.errorCode)
62
+ throw new error_1.KafkaTSApiError(result.errorCode, null, result);
63
+ result.responses.forEach((response) => {
64
+ response.partitions.forEach((partition) => {
65
+ if (partition.errorCode)
66
+ throw new error_1.KafkaTSApiError(partition.errorCode, null, result);
67
+ });
68
+ });
69
+ return result;
70
+ },
71
+ });
72
+ const decodeRecords = (decoder) => {
73
+ const size = decoder.readUVarInt() - 1;
74
+ if (size <= 0) {
75
+ return [];
76
+ }
77
+ const results = [];
78
+ while (decoder.getBufferLength() > decoder.getOffset() + 49) {
79
+ results.push({
80
+ baseOffset: decoder.readInt64(),
81
+ batchLength: decoder.readInt32(),
82
+ partitionLeaderEpoch: decoder.readInt32(),
83
+ magic: decoder.readInt8(),
84
+ crc: decoder.readUInt32(),
85
+ attributes: decoder.readInt16(),
86
+ lastOffsetDelta: decoder.readInt32(),
87
+ baseTimestamp: decoder.readInt64(),
88
+ maxTimestamp: decoder.readInt64(),
89
+ producerId: decoder.readInt64(),
90
+ producerEpoch: decoder.readInt16(),
91
+ baseSequence: decoder.readInt32(),
92
+ records: decoder.readRecords((record) => ({
93
+ attributes: record.readInt8(),
94
+ timestampDelta: record.readVarLong(),
95
+ offsetDelta: record.readVarInt(),
96
+ key: record.readVarIntString(),
97
+ value: record.readVarIntString(),
98
+ headers: record.readCompactArray((header) => ({
99
+ key: header.readVarIntString(),
100
+ value: header.readVarIntString(),
101
+ })),
102
+ })),
103
+ });
104
+ }
105
+ return results;
106
+ };
@@ -0,0 +1,21 @@
1
+ export declare const KEY_TYPE: {
2
+ GROUP: number;
3
+ TRANSACTION: number;
4
+ };
5
+ export declare const FIND_COORDINATOR: import("../utils/api").Api<{
6
+ keyType: number;
7
+ keys: string[];
8
+ }, {
9
+ _tag: void;
10
+ throttleTimeMs: number;
11
+ coordinators: {
12
+ key: string | null;
13
+ nodeId: number;
14
+ host: string;
15
+ port: number;
16
+ errorCode: number;
17
+ errorMessage: string | null;
18
+ _tag: void;
19
+ }[];
20
+ _tag2: void;
21
+ }>;
@@ -0,0 +1,39 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.FIND_COORDINATOR = exports.KEY_TYPE = void 0;
4
+ const api_1 = require("../utils/api");
5
+ const error_1 = require("../utils/error");
6
+ exports.KEY_TYPE = {
7
+ GROUP: 0,
8
+ TRANSACTION: 1,
9
+ };
10
+ exports.FIND_COORDINATOR = (0, api_1.createApi)({
11
+ apiKey: 10,
12
+ apiVersion: 4,
13
+ request: (encoder, data) => encoder
14
+ .writeUVarInt(0)
15
+ .writeInt8(data.keyType)
16
+ .writeCompactArray(data.keys, (encoder, key) => encoder.writeCompactString(key))
17
+ .writeUVarInt(0),
18
+ response: (decoder) => {
19
+ const result = {
20
+ _tag: decoder.readTagBuffer(),
21
+ throttleTimeMs: decoder.readInt32(),
22
+ coordinators: decoder.readCompactArray((decoder) => ({
23
+ key: decoder.readCompactString(),
24
+ nodeId: decoder.readInt32(),
25
+ host: decoder.readCompactString(),
26
+ port: decoder.readInt32(),
27
+ errorCode: decoder.readInt16(),
28
+ errorMessage: decoder.readCompactString(),
29
+ _tag: decoder.readTagBuffer(),
30
+ })),
31
+ _tag2: decoder.readTagBuffer(),
32
+ };
33
+ result.coordinators.forEach((coordinator) => {
34
+ if (coordinator.errorCode)
35
+ throw new error_1.KafkaTSApiError(coordinator.errorCode, coordinator.errorMessage, result);
36
+ });
37
+ return result;
38
+ },
39
+ });
@@ -0,0 +1,11 @@
1
+ export declare const HEARTBEAT: import("../utils/api").Api<{
2
+ groupId: string;
3
+ generationId: number;
4
+ memberId: string;
5
+ groupInstanceId: string | null;
6
+ }, {
7
+ _tag: void;
8
+ throttleTimeMs: number;
9
+ errorCode: number;
10
+ _tag2: void;
11
+ }>;
@@ -0,0 +1,27 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.HEARTBEAT = void 0;
4
+ const api_1 = require("../utils/api");
5
+ const error_1 = require("../utils/error");
6
+ exports.HEARTBEAT = (0, api_1.createApi)({
7
+ apiKey: 12,
8
+ apiVersion: 4,
9
+ request: (encoder, data) => encoder
10
+ .writeUVarInt(0)
11
+ .writeCompactString(data.groupId)
12
+ .writeInt32(data.generationId)
13
+ .writeCompactString(data.memberId)
14
+ .writeCompactString(data.groupInstanceId)
15
+ .writeUVarInt(0),
16
+ response: (decoder) => {
17
+ const result = {
18
+ _tag: decoder.readTagBuffer(),
19
+ throttleTimeMs: decoder.readInt32(),
20
+ errorCode: decoder.readInt16(),
21
+ _tag2: decoder.readTagBuffer(),
22
+ };
23
+ if (result.errorCode)
24
+ throw new error_1.KafkaTSApiError(result.errorCode, null, result);
25
+ return result;
26
+ },
27
+ });