@platformatic/kafka 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +270 -0
  3. package/dist/apis/admin/alter-client-quotas.d.ts +33 -0
  4. package/dist/apis/admin/alter-client-quotas.js +64 -0
  5. package/dist/apis/admin/alter-configs.d.ts +26 -0
  6. package/dist/apis/admin/alter-configs.js +57 -0
  7. package/dist/apis/admin/alter-partition-reassignments.d.ts +30 -0
  8. package/dist/apis/admin/alter-partition-reassignments.js +68 -0
  9. package/dist/apis/admin/alter-partition.d.ts +39 -0
  10. package/dist/apis/admin/alter-partition.js +87 -0
  11. package/dist/apis/admin/alter-replica-log-dirs.d.ts +26 -0
  12. package/dist/apis/admin/alter-replica-log-dirs.js +55 -0
  13. package/dist/apis/admin/alter-user-scram-credentials.d.ts +27 -0
  14. package/dist/apis/admin/alter-user-scram-credentials.js +60 -0
  15. package/dist/apis/admin/consumer-group-describe.d.ts +41 -0
  16. package/dist/apis/admin/consumer-group-describe.js +103 -0
  17. package/dist/apis/admin/create-acls.d.ts +24 -0
  18. package/dist/apis/admin/create-acls.js +55 -0
  19. package/dist/apis/admin/create-delegation-token.d.ts +24 -0
  20. package/dist/apis/admin/create-delegation-token.js +54 -0
  21. package/dist/apis/admin/create-partitions.d.ts +24 -0
  22. package/dist/apis/admin/create-partitions.js +54 -0
  23. package/dist/apis/admin/create-topics.d.ts +42 -0
  24. package/dist/apis/admin/create-topics.js +86 -0
  25. package/dist/apis/admin/delete-acls.d.ts +36 -0
  26. package/dist/apis/admin/delete-acls.js +82 -0
  27. package/dist/apis/admin/delete-groups.d.ts +14 -0
  28. package/dist/apis/admin/delete-groups.js +40 -0
  29. package/dist/apis/admin/delete-records.d.ts +27 -0
  30. package/dist/apis/admin/delete-records.js +59 -0
  31. package/dist/apis/admin/delete-topics.d.ts +21 -0
  32. package/dist/apis/admin/delete-topics.js +50 -0
  33. package/dist/apis/admin/describe-acls.d.ts +25 -0
  34. package/dist/apis/admin/describe-acls.js +66 -0
  35. package/dist/apis/admin/describe-client-quotas.d.ts +30 -0
  36. package/dist/apis/admin/describe-client-quotas.js +57 -0
  37. package/dist/apis/admin/describe-cluster.d.ts +23 -0
  38. package/dist/apis/admin/describe-cluster.js +50 -0
  39. package/dist/apis/admin/describe-configs.d.ts +38 -0
  40. package/dist/apis/admin/describe-configs.js +85 -0
  41. package/dist/apis/admin/describe-delegation-token.d.ts +31 -0
  42. package/dist/apis/admin/describe-delegation-token.js +62 -0
  43. package/dist/apis/admin/describe-groups.d.ts +28 -0
  44. package/dist/apis/admin/describe-groups.js +67 -0
  45. package/dist/apis/admin/describe-log-dirs.d.ts +32 -0
  46. package/dist/apis/admin/describe-log-dirs.js +75 -0
  47. package/dist/apis/admin/describe-producers.d.ts +33 -0
  48. package/dist/apis/admin/describe-producers.js +70 -0
  49. package/dist/apis/admin/describe-quorum.d.ts +50 -0
  50. package/dist/apis/admin/describe-quorum.js +116 -0
  51. package/dist/apis/admin/describe-topic-partitions.d.ts +42 -0
  52. package/dist/apis/admin/describe-topic-partitions.js +94 -0
  53. package/dist/apis/admin/describe-transactions.d.ts +24 -0
  54. package/dist/apis/admin/describe-transactions.js +59 -0
  55. package/dist/apis/admin/describe-user-scram-credentials.d.ts +26 -0
  56. package/dist/apis/admin/describe-user-scram-credentials.js +62 -0
  57. package/dist/apis/admin/envelope.d.ts +10 -0
  58. package/dist/apis/admin/envelope.js +32 -0
  59. package/dist/apis/admin/expire-delegation-token.d.ts +11 -0
  60. package/dist/apis/admin/expire-delegation-token.js +29 -0
  61. package/dist/apis/admin/incremental-alter-configs.d.ts +27 -0
  62. package/dist/apis/admin/incremental-alter-configs.js +58 -0
  63. package/dist/apis/admin/index.d.ts +37 -0
  64. package/dist/apis/admin/index.js +37 -0
  65. package/dist/apis/admin/list-groups.d.ts +18 -0
  66. package/dist/apis/admin/list-groups.js +43 -0
  67. package/dist/apis/admin/list-partition-reassignments.d.ts +27 -0
  68. package/dist/apis/admin/list-partition-reassignments.js +56 -0
  69. package/dist/apis/admin/list-transactions.d.ts +18 -0
  70. package/dist/apis/admin/list-transactions.js +45 -0
  71. package/dist/apis/admin/offset-delete.d.ts +26 -0
  72. package/dist/apis/admin/offset-delete.js +59 -0
  73. package/dist/apis/admin/renew-delegation-token.d.ts +11 -0
  74. package/dist/apis/admin/renew-delegation-token.js +29 -0
  75. package/dist/apis/admin/unregister-broker.d.ts +12 -0
  76. package/dist/apis/admin/unregister-broker.js +28 -0
  77. package/dist/apis/admin/update-features.d.ts +23 -0
  78. package/dist/apis/admin/update-features.js +60 -0
  79. package/dist/apis/consumer/consumer-group-heartbeat.d.ts +27 -0
  80. package/dist/apis/consumer/consumer-group-heartbeat.js +70 -0
  81. package/dist/apis/consumer/fetch.d.ts +46 -0
  82. package/dist/apis/consumer/fetch.js +121 -0
  83. package/dist/apis/consumer/heartbeat.d.ts +11 -0
  84. package/dist/apis/consumer/heartbeat.js +34 -0
  85. package/dist/apis/consumer/index.d.ts +9 -0
  86. package/dist/apis/consumer/index.js +9 -0
  87. package/dist/apis/consumer/join-group.d.ts +27 -0
  88. package/dist/apis/consumer/join-group.js +71 -0
  89. package/dist/apis/consumer/leave-group.d.ts +22 -0
  90. package/dist/apis/consumer/leave-group.js +57 -0
  91. package/dist/apis/consumer/list-offsets.d.ts +30 -0
  92. package/dist/apis/consumer/list-offsets.js +68 -0
  93. package/dist/apis/consumer/offset-commit.d.ts +29 -0
  94. package/dist/apis/consumer/offset-commit.js +68 -0
  95. package/dist/apis/consumer/offset-fetch.d.ts +37 -0
  96. package/dist/apis/consumer/offset-fetch.js +81 -0
  97. package/dist/apis/consumer/sync-group.d.ts +18 -0
  98. package/dist/apis/consumer/sync-group.js +49 -0
  99. package/dist/apis/definitions.d.ts +16 -0
  100. package/dist/apis/definitions.js +12 -0
  101. package/dist/apis/enumerations.d.ts +114 -0
  102. package/dist/apis/enumerations.js +78 -0
  103. package/dist/apis/index.d.ts +8 -0
  104. package/dist/apis/index.js +10 -0
  105. package/dist/apis/metadata/api-versions.d.ts +17 -0
  106. package/dist/apis/metadata/api-versions.js +41 -0
  107. package/dist/apis/metadata/find-coordinator.d.ts +19 -0
  108. package/dist/apis/metadata/find-coordinator.js +50 -0
  109. package/dist/apis/metadata/index.d.ts +3 -0
  110. package/dist/apis/metadata/index.js +3 -0
  111. package/dist/apis/metadata/metadata.d.ts +37 -0
  112. package/dist/apis/metadata/metadata.js +92 -0
  113. package/dist/apis/producer/add-offsets-to-txn.d.ts +10 -0
  114. package/dist/apis/producer/add-offsets-to-txn.js +34 -0
  115. package/dist/apis/producer/add-partitions-to-txn.d.ts +34 -0
  116. package/dist/apis/producer/add-partitions-to-txn.js +79 -0
  117. package/dist/apis/producer/end-txn.d.ts +10 -0
  118. package/dist/apis/producer/end-txn.js +34 -0
  119. package/dist/apis/producer/index.d.ts +6 -0
  120. package/dist/apis/producer/index.js +6 -0
  121. package/dist/apis/producer/init-producer-id.d.ts +21 -0
  122. package/dist/apis/producer/init-producer-id.js +38 -0
  123. package/dist/apis/producer/produce.d.ts +29 -0
  124. package/dist/apis/producer/produce.js +104 -0
  125. package/dist/apis/producer/txn-offset-commit.d.ts +29 -0
  126. package/dist/apis/producer/txn-offset-commit.js +77 -0
  127. package/dist/apis/security/index.d.ts +2 -0
  128. package/dist/apis/security/index.js +2 -0
  129. package/dist/apis/security/sasl-authenticate.d.ts +15 -0
  130. package/dist/apis/security/sasl-authenticate.js +30 -0
  131. package/dist/apis/security/sasl-handshake.d.ts +10 -0
  132. package/dist/apis/security/sasl-handshake.js +28 -0
  133. package/dist/apis/telemetry/get-telemetry-subscriptions.d.ts +18 -0
  134. package/dist/apis/telemetry/get-telemetry-subscriptions.js +46 -0
  135. package/dist/apis/telemetry/index.d.ts +3 -0
  136. package/dist/apis/telemetry/index.js +3 -0
  137. package/dist/apis/telemetry/list-client-metrics-resources.d.ts +14 -0
  138. package/dist/apis/telemetry/list-client-metrics-resources.js +32 -0
  139. package/dist/apis/telemetry/push-telemetry.d.ts +10 -0
  140. package/dist/apis/telemetry/push-telemetry.js +36 -0
  141. package/dist/clients/admin/admin.d.ts +18 -0
  142. package/dist/clients/admin/admin.js +322 -0
  143. package/dist/clients/admin/index.d.ts +3 -0
  144. package/dist/clients/admin/index.js +3 -0
  145. package/dist/clients/admin/options.d.ts +135 -0
  146. package/dist/clients/admin/options.js +81 -0
  147. package/dist/clients/admin/types.d.ts +56 -0
  148. package/dist/clients/admin/types.js +1 -0
  149. package/dist/clients/base/base.d.ts +48 -0
  150. package/dist/clients/base/base.js +242 -0
  151. package/dist/clients/base/index.d.ts +3 -0
  152. package/dist/clients/base/index.js +3 -0
  153. package/dist/clients/base/options.d.ts +115 -0
  154. package/dist/clients/base/options.js +59 -0
  155. package/dist/clients/base/types.d.ts +38 -0
  156. package/dist/clients/base/types.js +1 -0
  157. package/dist/clients/callbacks.d.ts +8 -0
  158. package/dist/clients/callbacks.js +42 -0
  159. package/dist/clients/consumer/consumer.d.ts +33 -0
  160. package/dist/clients/consumer/consumer.js +767 -0
  161. package/dist/clients/consumer/index.d.ts +5 -0
  162. package/dist/clients/consumer/index.js +5 -0
  163. package/dist/clients/consumer/messages-stream.d.ts +56 -0
  164. package/dist/clients/consumer/messages-stream.js +404 -0
  165. package/dist/clients/consumer/options.d.ts +521 -0
  166. package/dist/clients/consumer/options.js +177 -0
  167. package/dist/clients/consumer/topics-map.d.ts +8 -0
  168. package/dist/clients/consumer/topics-map.js +48 -0
  169. package/dist/clients/consumer/types.d.ts +74 -0
  170. package/dist/clients/consumer/types.js +11 -0
  171. package/dist/clients/index.d.ts +6 -0
  172. package/dist/clients/index.js +6 -0
  173. package/dist/clients/producer/index.d.ts +3 -0
  174. package/dist/clients/producer/index.js +3 -0
  175. package/dist/clients/producer/options.d.ts +122 -0
  176. package/dist/clients/producer/options.js +47 -0
  177. package/dist/clients/producer/producer.d.ts +13 -0
  178. package/dist/clients/producer/producer.js +272 -0
  179. package/dist/clients/producer/types.d.ts +29 -0
  180. package/dist/clients/producer/types.js +1 -0
  181. package/dist/clients/serde.d.ts +40 -0
  182. package/dist/clients/serde.js +49 -0
  183. package/dist/errors.d.ts +67 -0
  184. package/dist/errors.js +160 -0
  185. package/dist/index.d.ts +6 -0
  186. package/dist/index.js +10 -0
  187. package/dist/network/connection-pool.d.ts +11 -0
  188. package/dist/network/connection-pool.js +101 -0
  189. package/dist/network/connection.d.ts +49 -0
  190. package/dist/network/connection.js +319 -0
  191. package/dist/network/index.d.ts +2 -0
  192. package/dist/network/index.js +2 -0
  193. package/dist/protocol/apis.d.ts +2 -0
  194. package/dist/protocol/apis.js +191 -0
  195. package/dist/protocol/compression.d.ts +70 -0
  196. package/dist/protocol/compression.js +114 -0
  197. package/dist/protocol/crc32c.d.ts +2 -0
  198. package/dist/protocol/crc32c.js +83 -0
  199. package/dist/protocol/definitions.d.ts +12 -0
  200. package/dist/protocol/definitions.js +11 -0
  201. package/dist/protocol/dynamic-buffer.d.ts +65 -0
  202. package/dist/protocol/dynamic-buffer.js +557 -0
  203. package/dist/protocol/errors.d.ts +8 -0
  204. package/dist/protocol/errors.js +908 -0
  205. package/dist/protocol/index.d.ts +14 -0
  206. package/dist/protocol/index.js +14 -0
  207. package/dist/protocol/murmur2.d.ts +1 -0
  208. package/dist/protocol/murmur2.js +55 -0
  209. package/dist/protocol/reader.d.ts +58 -0
  210. package/dist/protocol/reader.js +296 -0
  211. package/dist/protocol/records.d.ts +110 -0
  212. package/dist/protocol/records.js +149 -0
  213. package/dist/protocol/sasl/plain.d.ts +3 -0
  214. package/dist/protocol/sasl/plain.js +3 -0
  215. package/dist/protocol/sasl/scram-sha.d.ts +28 -0
  216. package/dist/protocol/sasl/scram-sha.js +104 -0
  217. package/dist/protocol/varint.d.ts +12 -0
  218. package/dist/protocol/varint.js +36 -0
  219. package/dist/protocol/writer.d.ts +48 -0
  220. package/dist/protocol/writer.js +223 -0
  221. package/dist/utils.d.ts +20 -0
  222. package/dist/utils.js +127 -0
  223. package/package.json +75 -0
@@ -0,0 +1,3 @@
1
+ import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate.ts';
2
+ import { type Connection } from '../../network/connection.ts';
3
+ export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, username: string, password: string): Promise<SaslAuthenticateResponse>;
@@ -0,0 +1,3 @@
1
+ export function authenticate(authenticateAPI, connection, username, password) {
2
+ return authenticateAPI.async(connection, Buffer.from(['', username, password].join('\0')));
3
+ }
@@ -0,0 +1,28 @@
1
+ import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate.ts';
2
+ import { type Connection } from '../../network/connection.ts';
3
+ export interface ScramAlgorithmDefinition {
4
+ keyLength: number;
5
+ algorithm: string;
6
+ minIterations: number;
7
+ }
8
+ export declare const ScramAlgorithms: {
9
+ readonly 'SHA-256': {
10
+ readonly keyLength: 32;
11
+ readonly algorithm: "sha256";
12
+ readonly minIterations: 4096;
13
+ };
14
+ readonly 'SHA-512': {
15
+ readonly keyLength: 64;
16
+ readonly algorithm: "sha512";
17
+ readonly minIterations: 4096;
18
+ };
19
+ };
20
+ export type ScramAlgorithm = keyof typeof ScramAlgorithms;
21
+ export declare function createNonce(): string;
22
+ export declare function sanitizeString(str: string): string;
23
+ export declare function parseParameters(data: Buffer): Record<string, string>;
24
+ export declare function h(definition: ScramAlgorithmDefinition, data: string | Buffer): Buffer<ArrayBufferLike>;
25
+ export declare function hi(definition: ScramAlgorithmDefinition, password: string, salt: Buffer, iterations: number): Buffer<ArrayBufferLike>;
26
+ export declare function hmac(definition: ScramAlgorithmDefinition, key: Buffer, data: string | Buffer): Buffer;
27
+ export declare function xor(a: Buffer, b: Buffer): Buffer;
28
+ export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, username: string, password: string): Promise<SaslAuthenticateResponse>;
@@ -0,0 +1,104 @@
1
+ import { createHash, createHmac, pbkdf2Sync, randomBytes } from 'node:crypto';
2
+ import { AuthenticationError } from "../../errors.js";
3
+ const GS2_HEADER = 'n,,';
4
+ const GS2_HEADER_BASE64 = Buffer.from(GS2_HEADER).toString('base64');
5
+ const HMAC_CLIENT_KEY = 'Client Key';
6
+ const HMAC_SERVER_KEY = 'Server Key';
7
+ const PARAMETERS_PARSER = /([^=]+)=(.+)/;
8
+ export const ScramAlgorithms = {
9
+ 'SHA-256': {
10
+ keyLength: 32,
11
+ algorithm: 'sha256',
12
+ minIterations: 4096
13
+ },
14
+ 'SHA-512': {
15
+ keyLength: 64,
16
+ algorithm: 'sha512',
17
+ minIterations: 4096
18
+ }
19
+ };
20
+ export function createNonce() {
21
+ return randomBytes(16).toString('base64url');
22
+ }
23
+ export function sanitizeString(str) {
24
+ return str.replaceAll('=', '=3D').replace(',', '=2C');
25
+ }
26
+ export function parseParameters(data) {
27
+ const original = data.toString('utf-8');
28
+ return {
29
+ __original: original,
30
+ ...Object.fromEntries(original.split(',').map(param => param.match(PARAMETERS_PARSER).slice(1, 3)))
31
+ };
32
+ }
33
+ // h, hi, hmac and xor, are defined in https://datatracker.ietf.org/doc/html/rfc5802#section-2.2
34
+ export function h(definition, data) {
35
+ return createHash(definition.algorithm).update(data).digest();
36
+ }
37
+ export function hi(definition, password, salt, iterations) {
38
+ return pbkdf2Sync(password, salt, iterations, definition.keyLength, definition.algorithm);
39
+ }
40
+ export function hmac(definition, key, data) {
41
+ return createHmac(definition.algorithm, key).update(data).digest();
42
+ }
43
+ export function xor(a, b) {
44
+ if (a.byteLength !== b.byteLength) {
45
+ throw new AuthenticationError('Buffers must have the same length.');
46
+ }
47
+ const result = Buffer.allocUnsafe(a.length);
48
+ for (let i = 0; i < a.length; i++) {
49
+ result[i] = a[i] ^ b[i];
50
+ }
51
+ return result;
52
+ }
53
+ // Implements https://datatracker.ietf.org/doc/html/rfc5802#section-9
54
+ export async function authenticate(authenticateAPI, connection, algorithm, username, password) {
55
+ const definition = ScramAlgorithms[algorithm];
56
+ if (!definition) {
57
+ throw new AuthenticationError(`Unsupported SCRAM algorithm ${algorithm}`);
58
+ }
59
+ const clientNonce = createNonce();
60
+ const clientFirstMessageBare = `n=${sanitizeString(username)},r=${clientNonce}`;
61
+ // First of all, send the first message
62
+ const firstResponse = await authenticateAPI.async(connection, Buffer.from(`${GS2_HEADER}${clientFirstMessageBare}`));
63
+ const firstData = parseParameters(firstResponse.authBytes);
64
+ // Extract some parameters
65
+ const salt = Buffer.from(firstData.s, 'base64');
66
+ const iterations = parseInt(firstData.i, 10);
67
+ const serverNonce = firstData.r;
68
+ const serverFirstMessage = firstData.__original;
69
+ // Validate response
70
+ if (!serverNonce.startsWith(clientNonce)) {
71
+ throw new AuthenticationError('Server nonce does not start with client nonce.');
72
+ }
73
+ if (definition.minIterations > iterations) {
74
+ throw new AuthenticationError(`Algorithm ${algorithm} requires at least ${definition.minIterations} iterations, while ${iterations} were requested.`);
75
+ }
76
+ // SaltedPassword := Hi(Normalize(password), salt, i)
77
+ // ClientKey := HMAC(SaltedPassword, "Client Key")
78
+ // StoredKey := H(ClientKey)
79
+ // AuthMessage := ClientFirstMessageBare + "," ServerFirstMessage + "," + ClientFinalMessageWithoutProof
80
+ // ClientSignature := HMAC(StoredKey, AuthMessage)
81
+ // ClientProof := ClientKey XOR ClientSignature
82
+ // ServerKey := HMAC(SaltedPassword, "Server Key")
83
+ // ServerSignature := HMAC(ServerKey, AuthMessage)
84
+ const saltedPassword = hi(definition, password, salt, iterations);
85
+ const clientKey = hmac(definition, saltedPassword, HMAC_CLIENT_KEY);
86
+ const storedKey = h(definition, clientKey);
87
+ const clientFinalMessageWithoutProof = `c=${GS2_HEADER_BASE64},r=${serverNonce}`;
88
+ const authMessage = `${clientFirstMessageBare},${serverFirstMessage},${clientFinalMessageWithoutProof}`;
89
+ const clientSignature = hmac(definition, storedKey, authMessage);
90
+ const clientProof = xor(clientKey, clientSignature);
91
+ const serverKey = hmac(definition, saltedPassword, HMAC_SERVER_KEY);
92
+ const serverSignature = hmac(definition, serverKey, authMessage);
93
+ // Send the last message to the server
94
+ const lastResponse = await authenticateAPI.async(connection, Buffer.from(`${clientFinalMessageWithoutProof},p=${clientProof.toString('base64')}`));
95
+ const lastData = parseParameters(lastResponse.authBytes);
96
+ if (lastData.e) {
97
+ throw new AuthenticationError(lastData.e);
98
+ }
99
+ else if (lastData.v !== serverSignature.toString('base64')) {
100
+ throw new AuthenticationError('Invalid server signature.');
101
+ }
102
+ /* c8 ignore next 2 */
103
+ return lastResponse;
104
+ }
@@ -0,0 +1,12 @@
1
+ export declare const MOST_SIGNIFICANT_BIT_FLAG = 128;
2
+ export declare const MOST_SIGNIFICANT_BIT_FLAG_64 = 128n;
3
+ export declare const LEAST_SIGNIFICANT_7_BITS = 127;
4
+ export declare const LEAST_SIGNIFICANT_7_BITS_64 = 127n;
5
+ export declare const BITS_8PLUS_MASK: number;
6
+ export declare const BITS_8PLUS_MASK_64: bigint;
7
+ export declare function intZigZagEncode(value: number): number;
8
+ export declare function intZigZagDecode(value: number): number;
9
+ export declare function int64ZigZagEncode(value: bigint): bigint;
10
+ export declare function int64ZigZagDecode(value: bigint): bigint;
11
+ export declare function sizeOfUnsignedVarInt(value: number): number;
12
+ export declare function sizeOfUnsignedVarInt64(value: bigint): number;
@@ -0,0 +1,36 @@
1
+ export const MOST_SIGNIFICANT_BIT_FLAG = 0x80; // 128 or 1000 0000
2
+ export const MOST_SIGNIFICANT_BIT_FLAG_64 = 0x80n; // 128 or 1000 0000
3
+ export const LEAST_SIGNIFICANT_7_BITS = 0x7f; // 127 or 0111 1111
4
+ export const LEAST_SIGNIFICANT_7_BITS_64 = 0x7fn; // 127 or 0111 1111
5
+ // This is used in varint to check if there are any other bits set after the first 7 bits,
6
+ // which means it still needs more than a byte to represent the number in varint encoding
7
+ export const BITS_8PLUS_MASK = 0xffffffff - 0x7f;
8
+ export const BITS_8PLUS_MASK_64 = 0xffffffffn - 0x7fn;
9
+ export function intZigZagEncode(value) {
10
+ return (value << 1) ^ (value >> 31);
11
+ }
12
+ export function intZigZagDecode(value) {
13
+ return (value >> 1) ^ -(value & 1);
14
+ }
15
+ export function int64ZigZagEncode(value) {
16
+ return (value << 1n) ^ (value >> 31n);
17
+ }
18
+ export function int64ZigZagDecode(value) {
19
+ return (value >> 1n) ^ -(value & 1n);
20
+ }
21
+ export function sizeOfUnsignedVarInt(value) {
22
+ let bytes = 1;
23
+ while ((value & BITS_8PLUS_MASK) !== 0) {
24
+ bytes++;
25
+ value >>>= 7;
26
+ }
27
+ return bytes;
28
+ }
29
+ export function sizeOfUnsignedVarInt64(value) {
30
+ let bytes = 1;
31
+ while ((value & BITS_8PLUS_MASK_64) !== 0n) {
32
+ bytes++;
33
+ value >>= 7n;
34
+ }
35
+ return bytes;
36
+ }
@@ -0,0 +1,48 @@
1
+ import { type NullableString } from './definitions.ts';
2
+ import { DynamicBuffer } from './dynamic-buffer.ts';
3
+ export type ChildrenWriter = (w: Writer) => void;
4
+ export type EntryWriter<InputType> = (writer: Writer, entry: InputType, index: number) => void;
5
+ declare const instanceIdentifier: unique symbol;
6
+ export declare class Writer {
7
+ #private;
8
+ context: Record<string, any>;
9
+ [instanceIdentifier]: boolean;
10
+ static isWriter(target: any): boolean;
11
+ static create(): Writer;
12
+ constructor(bl?: DynamicBuffer);
13
+ get buffer(): Buffer;
14
+ get buffers(): Buffer[];
15
+ get dynamicBuffer(): DynamicBuffer;
16
+ get length(): number;
17
+ inspect(): string;
18
+ append(buffer: Buffer): this;
19
+ prepend(buffer: Buffer): this;
20
+ appendFrom(buffer: Writer | DynamicBuffer): this;
21
+ prependFrom(buffer: Writer | DynamicBuffer): this;
22
+ appendUnsignedInt8(value: number, append?: boolean): this;
23
+ appendUnsignedInt16(value: number, append?: boolean): this;
24
+ appendUnsignedInt32(value: number, append?: boolean): this;
25
+ appendUnsignedInt64(value: bigint, append?: boolean): this;
26
+ appendUnsignedVarInt(value: number, append?: boolean): this;
27
+ appendUnsignedVarInt64(value: bigint, append?: boolean): this;
28
+ appendInt8(value: number, append?: boolean): this;
29
+ appendInt16(value: number, append?: boolean): this;
30
+ appendInt32(value: number, append?: boolean): this;
31
+ appendInt64(value: bigint, append?: boolean): this;
32
+ appendFloat64(value: number, append?: boolean): this;
33
+ appendVarInt(value: number, append?: boolean): this;
34
+ appendVarInt64(value: bigint, append?: boolean): this;
35
+ appendBoolean(value: boolean): this;
36
+ appendString(value: NullableString, compact?: boolean, encoding?: BufferEncoding): this;
37
+ appendUUID(value: NullableString): this;
38
+ appendBytes(value: Buffer | undefined | null, compact?: boolean): this;
39
+ appendVarIntBytes(value: Buffer | null | undefined): this;
40
+ appendArray<InputType>(value: InputType[] | null | undefined, entryWriter: EntryWriter<InputType>, compact?: boolean, appendTrailingTaggedFields?: boolean): this;
41
+ appendMap<Key, Value>(value: Map<Key, Value> | null | undefined, entryWriter: EntryWriter<[Key, Value]>, compact?: boolean, appendTrailingTaggedFields?: boolean): this;
42
+ appendVarIntArray<InputType>(value: InputType[] | null | undefined, entryWriter: EntryWriter<InputType>): this;
43
+ appendVarIntMap<Key, Value>(value: Map<Key, Value> | null | undefined, entryWriter: EntryWriter<[Key, Value]>): this;
44
+ appendTaggedFields(_?: any[]): this;
45
+ prependLength(): this;
46
+ prependVarIntLength(): this;
47
+ }
48
+ export {};
@@ -0,0 +1,223 @@
1
+ import { humanize } from "../utils.js";
2
+ import { EMPTY_UUID } from "./definitions.js";
3
+ import { DynamicBuffer } from "./dynamic-buffer.js";
4
+ const instanceIdentifier = Symbol('plt.kafka.writer.instanceIdentifier');
5
+ export class Writer {
6
+ context;
7
+ #buffer;
8
+ [instanceIdentifier];
9
+ static isWriter(target) {
10
+ return target?.[instanceIdentifier] === true;
11
+ }
12
+ static create() {
13
+ return new Writer(new DynamicBuffer());
14
+ }
15
+ constructor(bl) {
16
+ this.#buffer = bl;
17
+ this.context = {};
18
+ this[instanceIdentifier] = true;
19
+ }
20
+ get buffer() {
21
+ return this.#buffer.buffer;
22
+ }
23
+ get buffers() {
24
+ return this.#buffer.buffers;
25
+ }
26
+ get dynamicBuffer() {
27
+ return this.#buffer;
28
+ }
29
+ get length() {
30
+ return this.#buffer.length;
31
+ }
32
+ inspect() {
33
+ return this.buffers.map((buffer, i) => humanize(`Buffer ${i}`, buffer)).join('\n');
34
+ }
35
+ append(buffer) {
36
+ this.#buffer.append(buffer);
37
+ return this;
38
+ }
39
+ prepend(buffer) {
40
+ this.#buffer.prepend(buffer);
41
+ return this;
42
+ }
43
+ appendFrom(buffer) {
44
+ this.#buffer.appendFrom(buffer?.dynamicBuffer ?? buffer);
45
+ return this;
46
+ }
47
+ prependFrom(buffer) {
48
+ this.#buffer.prependFrom(buffer?.dynamicBuffer ?? buffer);
49
+ return this;
50
+ }
51
+ appendUnsignedInt8(value, append = true) {
52
+ this.#buffer.writeUInt8(value, append);
53
+ return this;
54
+ }
55
+ appendUnsignedInt16(value, append = true) {
56
+ this.#buffer.writeUInt16BE(value, append);
57
+ return this;
58
+ }
59
+ appendUnsignedInt32(value, append = true) {
60
+ this.#buffer.writeUInt32BE(value, append);
61
+ return this;
62
+ }
63
+ appendUnsignedInt64(value, append = true) {
64
+ this.#buffer.writeBigUInt64BE(value, append);
65
+ return this;
66
+ }
67
+ appendUnsignedVarInt(value, append = true) {
68
+ this.#buffer.writeUnsignedVarInt(value, append);
69
+ return this;
70
+ }
71
+ appendUnsignedVarInt64(value, append = true) {
72
+ this.#buffer.writeUnsignedVarInt64(value, append);
73
+ return this;
74
+ }
75
+ appendInt8(value, append = true) {
76
+ this.#buffer.writeInt8(value, append);
77
+ return this;
78
+ }
79
+ appendInt16(value, append = true) {
80
+ this.#buffer.writeInt16BE(value, append);
81
+ return this;
82
+ }
83
+ appendInt32(value, append = true) {
84
+ this.#buffer.writeInt32BE(value, append);
85
+ return this;
86
+ }
87
+ appendInt64(value, append = true) {
88
+ this.#buffer.writeBigInt64BE(value, append);
89
+ return this;
90
+ }
91
+ // In Kafka float is actually a double
92
+ appendFloat64(value, append = true) {
93
+ this.#buffer.writeDoubleBE(value, append);
94
+ return this;
95
+ }
96
+ appendVarInt(value, append = true) {
97
+ this.#buffer.writeVarInt(value, append);
98
+ return this;
99
+ }
100
+ appendVarInt64(value, append = true) {
101
+ this.#buffer.writeVarInt64(value, append);
102
+ return this;
103
+ }
104
+ appendBoolean(value) {
105
+ return this.appendUnsignedInt8(value ? 1 : 0);
106
+ }
107
+ appendString(value, compact = true, encoding = 'utf-8') {
108
+ if (value == null) {
109
+ return compact ? this.appendUnsignedVarInt(0) : this.appendInt16(-1);
110
+ }
111
+ const buffer = Buffer.from(value, encoding);
112
+ if (compact) {
113
+ this.appendUnsignedVarInt(buffer.length + 1);
114
+ }
115
+ else {
116
+ this.appendInt16(buffer.length);
117
+ }
118
+ if (buffer.length) {
119
+ this.#buffer.append(buffer);
120
+ }
121
+ return this;
122
+ }
123
+ appendUUID(value) {
124
+ if (value == null) {
125
+ return this.append(EMPTY_UUID);
126
+ }
127
+ const buffer = Buffer.from(value.replaceAll('-', ''), 'hex');
128
+ this.#buffer.append(buffer);
129
+ return this;
130
+ }
131
+ appendBytes(value, compact = true) {
132
+ if (value == null) {
133
+ return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(-1);
134
+ }
135
+ if (compact) {
136
+ this.appendUnsignedVarInt(value.length + 1);
137
+ }
138
+ else {
139
+ this.appendInt32(value.length);
140
+ }
141
+ this.#buffer.append(value);
142
+ return this;
143
+ }
144
+ // Note that this does not follow the wire protocol specification and thus the length is not +1ed
145
+ appendVarIntBytes(value) {
146
+ if (value == null) {
147
+ return this.appendVarInt(0);
148
+ }
149
+ this.appendVarInt(value.length);
150
+ this.#buffer.append(value);
151
+ return this;
152
+ }
153
+ appendArray(value, entryWriter, compact = true, appendTrailingTaggedFields = true) {
154
+ if (value == null) {
155
+ return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(0);
156
+ }
157
+ const length = value.length;
158
+ if (compact) {
159
+ this.appendUnsignedVarInt(length + 1);
160
+ }
161
+ else {
162
+ this.appendInt32(length);
163
+ }
164
+ for (let i = 0; i < length; i++) {
165
+ entryWriter(this, value[i], i);
166
+ if (appendTrailingTaggedFields) {
167
+ this.appendTaggedFields();
168
+ }
169
+ }
170
+ return this;
171
+ }
172
+ appendMap(value, entryWriter, compact = true, appendTrailingTaggedFields = true) {
173
+ if (value == null) {
174
+ return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(0);
175
+ }
176
+ const length = value.size;
177
+ if (compact) {
178
+ this.appendUnsignedVarInt(length + 1);
179
+ }
180
+ else {
181
+ this.appendInt32(length);
182
+ }
183
+ let i = 0;
184
+ for (const entry of value) {
185
+ entryWriter(this, entry, i++);
186
+ if (appendTrailingTaggedFields) {
187
+ this.appendTaggedFields();
188
+ }
189
+ }
190
+ return this;
191
+ }
192
+ appendVarIntArray(value, entryWriter) {
193
+ if (value == null) {
194
+ return this.appendVarInt(0);
195
+ }
196
+ this.appendVarInt(value.length);
197
+ for (let i = 0; i < value.length; i++) {
198
+ entryWriter(this, value[i], i);
199
+ }
200
+ return this;
201
+ }
202
+ appendVarIntMap(value, entryWriter) {
203
+ if (value == null) {
204
+ return this.appendVarInt(0);
205
+ }
206
+ this.appendVarInt(value.size);
207
+ let i = 0;
208
+ for (const entry of value) {
209
+ entryWriter(this, entry, i++);
210
+ }
211
+ return this;
212
+ }
213
+ // TODO(ShogunPanda): Tagged fields are not supported yet
214
+ appendTaggedFields(_ = []) {
215
+ return this.appendInt8(0);
216
+ }
217
+ prependLength() {
218
+ return this.appendInt32(this.length, false);
219
+ }
220
+ prependVarIntLength() {
221
+ return this.appendVarInt(this.length, false);
222
+ }
223
+ }
@@ -0,0 +1,20 @@
1
+ import { Ajv } from 'ajv';
2
+ import debug from 'debug';
3
+ import { type DynamicBuffer } from './protocol/dynamic-buffer.ts';
4
+ export type DebugDumpLogger = (...args: any[]) => void;
5
+ export { setTimeout as sleep } from 'node:timers/promises';
6
+ export declare const ajv: Ajv;
7
+ export declare const loggers: Record<string, debug.Debugger>;
8
+ export declare class NumericMap extends Map<string, number> {
9
+ getWithDefault(key: string, fallback: number): number;
10
+ preIncrement(key: string, value: number, fallback: number): number;
11
+ postIncrement(key: string, value: number, fallback: number): number;
12
+ }
13
+ export declare function niceJoin(array: string[], lastSeparator?: string, separator?: string): string;
14
+ export declare function listErrorMessage(type: string[]): string;
15
+ export declare function enumErrorMessage(type: Record<string, unknown>, keysOnly?: boolean): string;
16
+ export declare function groupByProperty<Key, Value>(entries: Value[], property: keyof Value): [Key, Value[]][];
17
+ export declare function humanize(label: string, buffer: Buffer | DynamicBuffer): string;
18
+ export declare function setDebugDumpLogger(logger: DebugDumpLogger): void;
19
+ export declare function debugDump(...values: unknown[]): void;
20
+ export declare function executeWithTimeout<T = unknown>(promise: Promise<T>, timeout: number, timeoutValue?: string): Promise<T | string>;
package/dist/utils.js ADDED
@@ -0,0 +1,127 @@
1
+ import { Unpromise } from '@watchable/unpromise';
2
+ import { Ajv } from 'ajv';
3
+ import ajvErrors from 'ajv-errors';
4
+ import { setTimeout as sleep } from 'node:timers/promises';
5
+ import { inspect } from 'node:util';
6
+ import debug from 'debug';
7
+ export { setTimeout as sleep } from 'node:timers/promises';
8
+ export const ajv = new Ajv({ allErrors: true, coerceTypes: false, strict: true });
9
+ export const loggers = {
10
+ protocol: debug('plt:kafka:protocol'),
11
+ client: debug('plt:kafka:client'),
12
+ producer: debug('plt:kafka:producer'),
13
+ consumer: debug('plt:kafka:consumer'),
14
+ 'consumer:heartbeat': debug('plt:kafka:consumer:heartbeat'),
15
+ admin: debug('plt:kafka:admin')
16
+ };
17
+ // @ts-ignore
18
+ ajvErrors(ajv);
19
+ let debugDumpLogger = console.error;
20
+ ajv.addKeyword({
21
+ keyword: 'bigint',
22
+ validate(_, x) {
23
+ return typeof x === 'bigint';
24
+ },
25
+ error: {
26
+ message: 'must be bigint'
27
+ }
28
+ });
29
+ ajv.addKeyword({
30
+ keyword: 'map',
31
+ validate(_, x) {
32
+ return x instanceof Map;
33
+ },
34
+ error: {
35
+ message: 'must be Map'
36
+ }
37
+ });
38
+ ajv.addKeyword({
39
+ keyword: 'function',
40
+ validate(_, x) {
41
+ return typeof x === 'function';
42
+ },
43
+ error: {
44
+ message: 'must be function'
45
+ }
46
+ });
47
+ ajv.addKeyword({
48
+ keyword: 'buffer',
49
+ validate(_, x) {
50
+ return Buffer.isBuffer(x);
51
+ },
52
+ error: {
53
+ message: 'must be Buffer'
54
+ }
55
+ });
56
+ export class NumericMap extends Map {
57
+ getWithDefault(key, fallback) {
58
+ return this.get(key) ?? fallback;
59
+ }
60
+ preIncrement(key, value, fallback) {
61
+ let existing = this.getWithDefault(key, fallback);
62
+ existing += value;
63
+ this.set(key, existing);
64
+ return existing;
65
+ }
66
+ postIncrement(key, value, fallback) {
67
+ const existing = this.getWithDefault(key, fallback);
68
+ this.set(key, existing + value);
69
+ return existing;
70
+ }
71
+ }
72
+ export function niceJoin(array, lastSeparator = ' and ', separator = ', ') {
73
+ switch (array.length) {
74
+ case 0:
75
+ return '';
76
+ case 1:
77
+ return array[0];
78
+ case 2:
79
+ return array.join(lastSeparator);
80
+ default:
81
+ return array.slice(0, -1).join(separator) + lastSeparator + array.at(-1);
82
+ }
83
+ }
84
+ export function listErrorMessage(type) {
85
+ return `should be one of ${niceJoin(type, ' or ')}`;
86
+ }
87
+ export function enumErrorMessage(type, keysOnly = false) {
88
+ if (keysOnly) {
89
+ return `should be one of ${niceJoin(Object.keys(type), ' or ')}`;
90
+ }
91
+ return `should be one of ${niceJoin(Object.entries(type).map(([k, v]) => `${v} (${k})`), ' or ')}`;
92
+ }
93
+ export function groupByProperty(entries, property) {
94
+ const grouped = new Map();
95
+ const result = [];
96
+ for (const entry of entries) {
97
+ const value = entry[property];
98
+ let values = grouped.get(value);
99
+ if (!values) {
100
+ values = [];
101
+ grouped.set(value, values);
102
+ result.push([value, values]);
103
+ }
104
+ values.push(entry);
105
+ }
106
+ return result;
107
+ }
108
+ export function humanize(label, buffer) {
109
+ const formatted = buffer
110
+ .toString('hex')
111
+ .replaceAll(/(.{4})/g, '$1 ')
112
+ .trim();
113
+ return `${label} (${buffer.length} bytes): ${formatted}`;
114
+ }
115
+ export function setDebugDumpLogger(logger) {
116
+ debugDumpLogger = logger;
117
+ }
118
+ export function debugDump(...values) {
119
+ debugDumpLogger(new Date().toISOString(), ...values.map(v => (typeof v === 'string' ? v : inspect(v, false, 10))));
120
+ }
121
+ export async function executeWithTimeout(promise, timeout, timeoutValue = 'timeout') {
122
+ const ac = new AbortController();
123
+ return Unpromise.race([promise, sleep(timeout, timeoutValue, { signal: ac.signal, ref: false })]).then((value) => {
124
+ ac.abort();
125
+ return value;
126
+ });
127
+ }