kafka-ts 0.0.3-beta → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +72 -8
  2. package/dist/api/api-versions.d.ts +9 -0
  3. package/{src/api/api-versions.ts → dist/api/api-versions.js} +8 -5
  4. package/dist/api/create-topics.d.ts +38 -0
  5. package/dist/api/create-topics.js +53 -0
  6. package/dist/api/delete-topics.d.ts +18 -0
  7. package/dist/api/delete-topics.js +33 -0
  8. package/dist/api/fetch.d.ts +84 -0
  9. package/dist/api/fetch.js +142 -0
  10. package/dist/api/find-coordinator.d.ts +21 -0
  11. package/{src/api/find-coordinator.ts → dist/api/find-coordinator.js} +14 -14
  12. package/dist/api/heartbeat.d.ts +11 -0
  13. package/dist/api/heartbeat.js +27 -0
  14. package/dist/api/index.d.ts +576 -0
  15. package/{src/api/index.ts → dist/api/index.js} +42 -41
  16. package/dist/api/init-producer-id.d.ts +13 -0
  17. package/dist/api/init-producer-id.js +29 -0
  18. package/dist/api/join-group.d.ts +34 -0
  19. package/dist/api/join-group.js +51 -0
  20. package/dist/api/leave-group.d.ts +19 -0
  21. package/dist/api/leave-group.js +39 -0
  22. package/dist/api/list-offsets.d.ts +29 -0
  23. package/dist/api/list-offsets.js +48 -0
  24. package/dist/api/metadata.d.ts +40 -0
  25. package/{src/api/metadata.ts → dist/api/metadata.js} +18 -26
  26. package/dist/api/offset-commit.d.ts +28 -0
  27. package/dist/api/offset-commit.js +48 -0
  28. package/dist/api/offset-fetch.d.ts +31 -0
  29. package/dist/api/offset-fetch.js +55 -0
  30. package/dist/api/produce.d.ts +54 -0
  31. package/{src/api/produce.ts → dist/api/produce.js} +55 -102
  32. package/dist/api/sasl-authenticate.d.ts +11 -0
  33. package/dist/api/sasl-authenticate.js +23 -0
  34. package/dist/api/sasl-handshake.d.ts +6 -0
  35. package/dist/api/sasl-handshake.js +19 -0
  36. package/dist/api/sync-group.d.ts +24 -0
  37. package/dist/api/sync-group.js +36 -0
  38. package/dist/auth/index.d.ts +2 -0
  39. package/dist/auth/index.js +8 -0
  40. package/dist/auth/plain.d.ts +5 -0
  41. package/dist/auth/plain.js +12 -0
  42. package/dist/auth/scram.d.ts +9 -0
  43. package/dist/auth/scram.js +40 -0
  44. package/dist/broker.d.ts +30 -0
  45. package/dist/broker.js +55 -0
  46. package/dist/client.d.ts +22 -0
  47. package/dist/client.js +36 -0
  48. package/dist/cluster.d.ts +27 -0
  49. package/dist/cluster.js +70 -0
  50. package/dist/cluster.test.d.ts +1 -0
  51. package/{src/cluster.test.ts → dist/cluster.test.js} +87 -113
  52. package/dist/codecs/gzip.d.ts +2 -0
  53. package/dist/codecs/gzip.js +8 -0
  54. package/dist/codecs/index.d.ts +2 -0
  55. package/dist/codecs/index.js +17 -0
  56. package/dist/codecs/none.d.ts +2 -0
  57. package/dist/codecs/none.js +7 -0
  58. package/dist/codecs/types.d.ts +5 -0
  59. package/dist/codecs/types.js +2 -0
  60. package/dist/connection.d.ts +26 -0
  61. package/dist/connection.js +175 -0
  62. package/dist/consumer/consumer-group.d.ts +41 -0
  63. package/dist/consumer/consumer-group.js +215 -0
  64. package/dist/consumer/consumer-metadata.d.ts +7 -0
  65. package/dist/consumer/consumer-metadata.js +14 -0
  66. package/dist/consumer/consumer.d.ts +44 -0
  67. package/dist/consumer/consumer.js +225 -0
  68. package/dist/consumer/fetch-manager.d.ts +33 -0
  69. package/dist/consumer/fetch-manager.js +140 -0
  70. package/dist/consumer/fetcher.d.ts +25 -0
  71. package/dist/consumer/fetcher.js +64 -0
  72. package/dist/consumer/offset-manager.d.ts +22 -0
  73. package/dist/consumer/offset-manager.js +66 -0
  74. package/dist/consumer/processor.d.ts +19 -0
  75. package/dist/consumer/processor.js +59 -0
  76. package/dist/distributors/assignments-to-replicas.d.ts +16 -0
  77. package/{src/distributors/assignments-to-replicas.ts → dist/distributors/assignments-to-replicas.js} +15 -41
  78. package/dist/distributors/assignments-to-replicas.test.d.ts +1 -0
  79. package/dist/distributors/assignments-to-replicas.test.js +40 -0
  80. package/dist/distributors/messages-to-topic-partition-leaders.d.ts +17 -0
  81. package/dist/distributors/messages-to-topic-partition-leaders.js +15 -0
  82. package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts +1 -0
  83. package/dist/distributors/messages-to-topic-partition-leaders.test.js +30 -0
  84. package/dist/distributors/partitioner.d.ts +7 -0
  85. package/dist/distributors/partitioner.js +23 -0
  86. package/dist/index.d.ts +9 -0
  87. package/dist/index.js +26 -0
  88. package/dist/metadata.d.ts +24 -0
  89. package/dist/metadata.js +106 -0
  90. package/dist/producer/producer.d.ts +24 -0
  91. package/dist/producer/producer.js +131 -0
  92. package/{src/types.ts → dist/types.d.ts} +4 -4
  93. package/dist/types.js +2 -0
  94. package/{src/utils/api.ts → dist/utils/api.d.ts} +2 -4
  95. package/dist/utils/api.js +5 -0
  96. package/dist/utils/crypto.d.ts +8 -0
  97. package/dist/utils/crypto.js +18 -0
  98. package/dist/utils/decoder.d.ts +30 -0
  99. package/{src/utils/decoder.ts → dist/utils/decoder.js} +41 -57
  100. package/dist/utils/delay.d.ts +1 -0
  101. package/dist/utils/delay.js +5 -0
  102. package/dist/utils/encoder.d.ts +28 -0
  103. package/{src/utils/encoder.ts → dist/utils/encoder.js} +50 -66
  104. package/dist/utils/error.d.ts +11 -0
  105. package/dist/utils/error.js +27 -0
  106. package/dist/utils/logger.d.ts +9 -0
  107. package/dist/utils/logger.js +32 -0
  108. package/dist/utils/memo.d.ts +1 -0
  109. package/{src/utils/memo.ts → dist/utils/memo.js} +7 -3
  110. package/dist/utils/murmur2.d.ts +3 -0
  111. package/dist/utils/murmur2.js +40 -0
  112. package/dist/utils/retrier.d.ts +10 -0
  113. package/dist/utils/retrier.js +22 -0
  114. package/dist/utils/tracer.d.ts +5 -0
  115. package/dist/utils/tracer.js +39 -0
  116. package/package.json +11 -2
  117. package/.github/workflows/release.yml +0 -17
  118. package/.prettierrc +0 -8
  119. package/certs/ca.crt +0 -29
  120. package/certs/ca.key +0 -52
  121. package/certs/ca.srl +0 -1
  122. package/certs/kafka.crt +0 -29
  123. package/certs/kafka.csr +0 -26
  124. package/certs/kafka.key +0 -52
  125. package/certs/kafka.keystore.jks +0 -0
  126. package/certs/kafka.truststore.jks +0 -0
  127. package/docker-compose.yml +0 -104
  128. package/examples/package-lock.json +0 -31
  129. package/examples/package.json +0 -14
  130. package/examples/src/client.ts +0 -9
  131. package/examples/src/consumer.ts +0 -18
  132. package/examples/src/create-topic.ts +0 -44
  133. package/examples/src/producer.ts +0 -24
  134. package/examples/src/replicator.ts +0 -25
  135. package/examples/src/utils/delay.ts +0 -1
  136. package/examples/src/utils/json.ts +0 -1
  137. package/examples/tsconfig.json +0 -7
  138. package/log4j.properties +0 -95
  139. package/scripts/generate-certs.sh +0 -24
  140. package/src/__snapshots__/request-handler.test.ts.snap +0 -978
  141. package/src/api/create-topics.ts +0 -78
  142. package/src/api/delete-topics.ts +0 -42
  143. package/src/api/fetch.ts +0 -143
  144. package/src/api/heartbeat.ts +0 -33
  145. package/src/api/init-producer-id.ts +0 -35
  146. package/src/api/join-group.ts +0 -67
  147. package/src/api/leave-group.ts +0 -48
  148. package/src/api/list-offsets.ts +0 -65
  149. package/src/api/offset-commit.ts +0 -67
  150. package/src/api/offset-fetch.ts +0 -74
  151. package/src/api/sasl-authenticate.ts +0 -21
  152. package/src/api/sasl-handshake.ts +0 -16
  153. package/src/api/sync-group.ts +0 -54
  154. package/src/broker.ts +0 -74
  155. package/src/client.ts +0 -47
  156. package/src/cluster.ts +0 -87
  157. package/src/connection.ts +0 -143
  158. package/src/consumer/consumer-group.ts +0 -209
  159. package/src/consumer/consumer-metadata.ts +0 -14
  160. package/src/consumer/consumer.ts +0 -231
  161. package/src/consumer/fetch-manager.ts +0 -179
  162. package/src/consumer/fetcher.ts +0 -57
  163. package/src/consumer/offset-manager.ts +0 -93
  164. package/src/consumer/processor.ts +0 -47
  165. package/src/distributors/assignments-to-replicas.test.ts +0 -43
  166. package/src/distributors/messages-to-topic-partition-leaders.test.ts +0 -32
  167. package/src/distributors/messages-to-topic-partition-leaders.ts +0 -19
  168. package/src/index.ts +0 -4
  169. package/src/metadata.ts +0 -122
  170. package/src/producer/producer.ts +0 -132
  171. package/src/utils/debug.ts +0 -9
  172. package/src/utils/delay.ts +0 -1
  173. package/src/utils/error.ts +0 -21
  174. package/src/utils/retrier.ts +0 -39
  175. package/src/utils/tracer.ts +0 -31
  176. package/tsconfig.json +0 -17
package/certs/kafka.key DELETED
@@ -1,52 +0,0 @@
1
- -----BEGIN PRIVATE KEY-----
2
- MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDPMfQ6rqXKOj51
3
- /MTiujAGpXgRzi82JdUKCa3Vl3ReDv2PceFpVl/fi0uJvEDUzipkgB7wTycuRIOC
4
- 8/AiHy6co/VoCf4/j97sTOHeV4kwV8+o//s5UApy3HC34o00ZwcP1yHJkN7Cnn1Z
5
- cA/Txulv1RfNqjhWaVzsxcs5rsb8C3+Wn4NbttBo7YXMf1wDPHYNzRTk2agiE3vH
6
- GqMrNAvrGmYmWhf+2a+LghG8EdBatXV0GnhHA+QhqWCq84CyzfOecFndq6niZx6V
7
- 3yj4yHDCjG+TefGV2SMuYgWSxAlV1g7lrzV63H910rhzLEjTbfET6n6LhCDZUCZX
8
- U1JaOO+B186+7AUN75sVYHwKjHCFgivNmpnwGy4FiDfhs2V+3CoFGQEYDgU3PaYD
9
- s0BFrDPnGeJtb/4bAaO8dzwaYjbb+rrks7YxzlHYJO51kAxCEZgU7OfL2vh3CdWK
10
- 7nBFy+rYjQGHM5hSI2pF+xQttRlA7qvYcPD+OIGy92D34QLlSnn97zZ9h+2lateK
11
- bMOH915lQd1Ac280+ZEEwp2kEB3TFyH3e+iQnXwcaqmQGJsD4Kue3hn0sG6LksMM
12
- zZstvVxAoWk9Mrp8s1+UG919Zc3p7/8ZovXtN+h1VG7NdKop4ZYretzyY50OY4vy
13
- ao58GhPMO+O5n7d8I1YFTTfQzR2ZlwIDAQABAoICAACl6lqoozIfvoRF9yOya2YJ
14
- 7Hc5yuIDekgewaFyaNXGfDqPo6HduE0NtX3HzfgccPuBfxdHWBsH8dVeB/lDWbT5
15
- ulHJ/jaar5rTnr1ATtiM3cRskI+cVXwIfn3EwqZQUDBXfu6PenLnZgL6m8IQ1Dga
16
- VYm6J5T1nWnMMJeMmWi+bHa/Jv0JdFtvqsR5/YKxFLl0U8VnfOQmZXO4/Jus/QCO
17
- Q5WJIJwWeDsafA23jIxMFwL29jtaosMMMZY3TsbAaCuttEpCODTSF8FmQk2M6vVg
18
- 1RSEBCJw3HBPtC7CWDkhg0hqpOMuf2OR9iu+DrhClsd2K1EqZ6PmYUhSpWZM2Urw
19
- PCVaTl1ZSkogJVv2OmrpGWm3qfMJCOr1VYk5onOsANIZUeo9wYYZNcA7+ObR0mS2
20
- tZrP+P5uP4FCVKA94+s/UOT5rkRw/jGJ8M776aOTXrrmJBMiUV1ExbmLbFmpX6FE
21
- VInQAdsN1Shg8drUttHL3M1W5TXldAn/BBMiyFSbQu45EQqj+U+na/ilniHJeYzR
22
- 5vjJc61FA8ku1FQ4uYLe58xggVgVizl2/sdwPXVxUgbQ+YfnKtQNxvsY9fpk+dm6
23
- x5bwqBCKiu6jQ0PN+/pxpc14UZmU8kvh+CZJ8G/jZzaovAINwxtS1pIzyXuTfLcT
24
- WVBmgZpduYYEHKCg1DepAoIBAQD+vqddxTQmIgN8LH+3E/xBnrEsO3BHNulkMYwK
25
- /RiPtokBAO0zWSHqhcClI+mqLX0wJmHSv5quXOWm37DBE0QAhFFXq+ZEDd2aPlXq
26
- zU0g7Cmm3qhnPitGxNlvS3Gyf7OKuUQuvi2bRmKDxRHZ/idTn3OZaKAiMda7rZBR
27
- ZT++zqRQ5dXjIO0ANIVgn2e4kW7CEo2dE0b7NNJ1RCrenk55MffOoVexWxDwbksM
28
- Lt3kxpd3s7xCbVPTACJKX3aqxhq9sUDDHWQ/RwcJXnMb3N+V01W2fOjGcudHDPYe
29
- OqAC1FaDF77hQpoeZZVy0fLfAJVWYEZV15oYkcVCx0idVSu1AoIBAQDQN1GvEhOp
30
- dM3p8v98OWWkDLcvGPF8m/UdKdnQxazpLqye2gDOelBZjtlBJByqWCoYymEPIAbk
31
- NzhsmiMNlQ+RI3rqGSDBqvagu3TpJp7DVWrPbzHyex/iSv18xxBiu4P3SVURwRcv
32
- Nls4lBJnZ+MOcAtOupQ0XQke4+rcdTMq17OIwWkveYRNIk89HqAcqaEEbC3wAR3V
33
- FMTj0As70nFdagbKWTfQRT+m4hCTa0u5H40Ej5U3ZNgV4kq91N8XRpf9i2FHpZcS
34
- GZMSnizxjJpt1/M5z06wDN7kIdS6rjFn/vHBPVlCdxGvbX480BAi/AicmP4gqRiU
35
- RTLCdJpZL3ebAoIBAF8hn6ra7CCZbye+Dmu2IOv4cT2NbN6LWRk7j4rC/ZsdlYg0
36
- amBUlt/X1xtUj/wnRKroinuDZiZbNr/cpUtVAH4Y66+L2E/txaXGRZuL05PJ3bwd
37
- RlJX2x1wgaqYR/tayRezHA1ApBNqc3u16na3/MOzth3B/psGdaRcqppa7zU4QhCb
38
- 5AYP07HR60UsAjXgybXb69qOymEhVr7IGCJOiAumZ15YzkDDO04uQgRotCCtMP9Y
39
- taMw5K7VS/juMYiQtofEW+UBYuBmi/Apv17WWwC+jTG678q8OjZEVtzvE7Y6yZvn
40
- EDdbAeb8FJVuZf51X0+EYH/KJheqyDmp5iorFkkCggEBAJLz2BTFn1bj90h8llBo
41
- ehH0X31AD3+w2qhAl1g/i6/vnylmF1ei//6VUUQmZozbOKrSevof0n1Tc+FCUCcP
42
- 4zK+Zcu+/F1zeikq5HKBLWATcxUAF7JZr8EYyBs+q5PjYFoxSXcEXoaVyIAAEZUl
43
- vxy7e1+PU3Y5i2xf06zQktJT9oQ4bTJygyulYWNzr/FKzze92hYf05xF0kgu1x7b
44
- geM6J5y8pyW5GhmAR86j1YfMesyTQe8cn12inrmn+eQFkNQDgR7+AcdmgWSMeo/o
45
- ej24QhO0+oZKLy23qtougvQw/RMzZ0H/o12623O0LQ9j7fgICqOmdJCA3jhb72uu
46
- kq0CggEAfJceJjzSWN1wmr40DwAwgmHuBWh2y+0m/kyMGqRuPEIHTL2U8wCXIdmb
47
- ugUjIbsb1lVdg3SdPnbV6X1CUuRoMd6XJ9cwR2NDgMr0DbCZsMq+g3dVKIPUFAoU
48
- hB5Ai2fvrSKTdJsKHklSEoAP3Jf9GW8TXANiag8UhKYKN1Mc0kNx34LWiIjElneN
49
- oRyE5pSnZcNeSkZFx2BYMd2YASHNbph8Nd9AIbV53CQ3yMD4mwDxnerkEvRsFKZd
50
- FR7c8kID+YBN2D7I1HUN3uflt/VNZGPc1ZjFWk+UyXBf04TPRAG4C0qEshQC3nwA
51
- 0EaZY4cBQwWFep18Shu5QuXvsTbHmQ==
52
- -----END PRIVATE KEY-----
Binary file
Binary file
@@ -1,104 +0,0 @@
1
- # kafka with raft:
2
- services:
3
- kafka-0:
4
- container_name: kafka-0
5
- image: apache/kafka:3.7.1
6
- ports:
7
- - '9092:9092'
8
- - '29092:29092'
9
- environment:
10
- KAFKA_NODE_ID: 0
11
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
12
- KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
13
- KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
14
- KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092,INTERBROKER://kafka-0:19092'
15
- KAFKA_PROCESS_ROLES: 'broker,controller'
16
- KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
17
- KAFKA_LISTENERS: 'EXTERNAL://:9092,INTERBROKER://:19092,CONTROLLER://:29092'
18
- KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
19
- KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
20
- CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
21
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
22
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
23
- KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
24
- KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
25
- KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
26
- KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
27
- KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
28
- KAFKA_SSL_KEY_PASSWORD: 'password'
29
- KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
30
- KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
31
- KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
32
- KAFKA_SSL_CLIENT_AUTH: 'required'
33
- volumes:
34
- - ./log4j.properties:/etc/kafka/docker/log4j.properties
35
- - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
36
- - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
37
- kafka-1:
38
- container_name: kafka-1
39
- image: apache/kafka:3.7.1
40
- ports:
41
- - '9093:9093'
42
- - '29093:29093'
43
- environment:
44
- KAFKA_NODE_ID: 1
45
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
46
- KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
47
- KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
48
- KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9093,INTERBROKER://kafka-1:19093'
49
- KAFKA_PROCESS_ROLES: 'broker,controller'
50
- KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
51
- KAFKA_LISTENERS: 'EXTERNAL://:9093,INTERBROKER://:19093,CONTROLLER://:29093'
52
- KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
53
- KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
54
- CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
55
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
56
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
57
- KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
58
- KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
59
- KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
60
- KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
61
- KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
62
- KAFKA_SSL_KEY_PASSWORD: 'password'
63
- KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
64
- KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
65
- KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
66
- KAFKA_SSL_CLIENT_AUTH: 'required'
67
- volumes:
68
- - ./log4j.properties:/etc/kafka/docker/log4j.properties
69
- - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
70
- - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
71
- kafka-2:
72
- container_name: kafka-2
73
- image: apache/kafka:3.7.1
74
- ports:
75
- - '9094:9094'
76
- - '29094:29094'
77
- environment:
78
- KAFKA_NODE_ID: 2
79
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
80
- KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
81
- KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
82
- KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9094,INTERBROKER://kafka-2:19094'
83
- KAFKA_PROCESS_ROLES: 'broker,controller'
84
- KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
85
- KAFKA_LISTENERS: 'EXTERNAL://:9094,INTERBROKER://:19094,CONTROLLER://:29094'
86
- KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
87
- KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
88
- CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
89
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
90
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
91
- KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
92
- KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
93
- KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
94
- KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
95
- KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
96
- KAFKA_SSL_KEY_PASSWORD: 'password'
97
- KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
98
- KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
99
- KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
100
- KAFKA_SSL_CLIENT_AUTH: 'required'
101
- volumes:
102
- - ./log4j.properties:/etc/kafka/docker/log4j.properties
103
- - ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
104
- - ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
@@ -1,31 +0,0 @@
1
- {
2
- "name": "examples",
3
- "version": "1.0.0",
4
- "lockfileVersion": 3,
5
- "requires": true,
6
- "packages": {
7
- "": {
8
- "name": "examples",
9
- "version": "1.0.0",
10
- "license": "ISC",
11
- "dependencies": {
12
- "kafka-ts": "file:../"
13
- }
14
- },
15
- "..": {
16
- "name": "kafka-ts",
17
- "version": "0.0.1-beta",
18
- "license": "MIT",
19
- "devDependencies": {
20
- "@types/node": "^20.12.12",
21
- "prettier": "^3.2.5",
22
- "typescript": "^5.4.5",
23
- "vitest": "^1.6.0"
24
- }
25
- },
26
- "node_modules/kafka-ts": {
27
- "resolved": "..",
28
- "link": true
29
- }
30
- }
31
- }
@@ -1,14 +0,0 @@
1
- {
2
- "name": "examples",
3
- "version": "1.0.0",
4
- "description": "",
5
- "main": "dist/replicator.js",
6
- "scripts": {
7
- "test": "echo \"Error: no test specified\" && exit 1"
8
- },
9
- "dependencies": {
10
- "kafka-ts": "file:../"
11
- },
12
- "author": "",
13
- "license": "ISC"
14
- }
@@ -1,9 +0,0 @@
1
- import { readFileSync } from 'fs';
2
- import { createKafkaClient } from 'kafka-ts';
3
-
4
- export const kafka = createKafkaClient({
5
- clientId: 'examples',
6
- bootstrapServers: [{ host: 'localhost', port: 9092 }],
7
- sasl: { mechanism: 'PLAIN', username: 'admin', password: 'admin' },
8
- ssl: { ca: readFileSync('../certs/ca.crt').toString() },
9
- });
@@ -1,18 +0,0 @@
1
- import { kafka } from './client';
2
-
3
- (async () => {
4
- const consumer = await kafka.startConsumer({
5
- groupId: 'example-group',
6
- groupInstanceId: 'example-group-instance',
7
- topics: ['my-topic'],
8
- onBatch: (batch) => {
9
- console.log(batch);
10
- },
11
- granularity: 'broker',
12
- concurrency: 10,
13
- });
14
-
15
- process.on('SIGINT', async () => {
16
- await consumer.close();
17
- });
18
- })();
@@ -1,44 +0,0 @@
1
- import { API, API_ERROR, KafkaTSApiError } from 'kafka-ts';
2
- import { kafka } from './client';
3
-
4
- (async () => {
5
- const cluster = kafka.createCluster();
6
- await cluster.connect();
7
-
8
- const { controllerId } = await cluster.sendRequest(API.METADATA, {
9
- allowTopicAutoCreation: false,
10
- includeTopicAuthorizedOperations: false,
11
- topics: [],
12
- });
13
-
14
- try {
15
- await cluster.sendRequestToNode(controllerId)(API.CREATE_TOPICS, {
16
- validateOnly: false,
17
- timeoutMs: 10_000,
18
- topics: [
19
- {
20
- name: 'my-topic',
21
- numPartitions: 10,
22
- replicationFactor: 3,
23
- assignments: [],
24
- configs: [],
25
- },
26
- ],
27
- });
28
- } catch (error) {
29
- if ((error as KafkaTSApiError).errorCode === API_ERROR.TOPIC_ALREADY_EXISTS) {
30
- return;
31
- }
32
- throw error;
33
- }
34
-
35
- const metadata = await cluster.sendRequestToNode(controllerId)(API.METADATA, {
36
- allowTopicAutoCreation: false,
37
- includeTopicAuthorizedOperations: false,
38
- topics: [{ id: null, name: 'my-topic' }],
39
- });
40
-
41
- console.log(metadata);
42
-
43
- await cluster.disconnect();
44
- })();
@@ -1,24 +0,0 @@
1
- import { createInterface } from 'readline';
2
- import { kafka } from './client';
3
-
4
- const producer = kafka.createProducer({ allowTopicAutoCreation: true });
5
-
6
- const rl = createInterface({ input: process.stdin, output: process.stdout });
7
-
8
- process.stdout.write('> ');
9
- rl.on('line', async (line) => {
10
- await producer.send([
11
- {
12
- topic: 'example-topic-f',
13
- key: null,
14
- value: line,
15
- partition: 0,
16
- },
17
- ]);
18
- process.stdout.write('> ');
19
- });
20
-
21
- process.on('SIGINT', async () => {
22
- rl.close();
23
- await producer.close();
24
- });
@@ -1,25 +0,0 @@
1
- import { kafka } from './client';
2
-
3
- (async () => {
4
- const topic = 'example-topic';
5
-
6
- const producer = kafka.createProducer({ allowTopicAutoCreation: true });
7
- const consumer = await kafka.startConsumer({
8
- topics: [topic],
9
- onBatch: async (messages) => {
10
- await producer.send(
11
- messages.map((message) => ({
12
- ...message,
13
- headers: { 'X-Replicated': 'true' },
14
- topic: `${message.topic}-replicated`,
15
- offset: 0n,
16
- })),
17
- );
18
- console.log(`Replicated ${messages.length} messages`);
19
- },
20
- });
21
- process.on('SIGINT', async () => {
22
- await consumer.close();
23
- await producer.close();
24
- });
25
- })();
@@ -1 +0,0 @@
1
- export const delay = (delayMs: number) => new Promise<void>((resolve) => setTimeout(resolve, delayMs));
@@ -1 +0,0 @@
1
- export const serializer = (_: string, value: unknown) => (typeof value === 'bigint' ? value.toString() : value);
@@ -1,7 +0,0 @@
1
- {
2
- "extends": "../tsconfig.json",
3
- "compilerOptions": {
4
- "outDir": "dist",
5
- "inlineSourceMap": true
6
- }
7
- }
package/log4j.properties DELETED
@@ -1,95 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one or more
2
- # contributor license agreements. See the NOTICE file distributed with
3
- # this work for additional information regarding copyright ownership.
4
- # The ASF licenses this file to You under the Apache License, Version 2.0
5
- # (the "License"); you may not use this file except in compliance with
6
- # the License. You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
-
16
- # Unspecified loggers and loggers with additivity=true output to server.log and stdout
17
- # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
18
- log4j.rootLogger=INFO, stdout, kafkaAppender
19
-
20
- log4j.appender.stdout=org.apache.log4j.ConsoleAppender
21
- log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
22
- log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
23
-
24
- log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
25
- log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH
26
- log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
27
- log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
28
- log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
29
-
30
- log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender
31
- log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH
32
- log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
33
- log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
34
- log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
35
-
36
- log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender
37
- log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH
38
- log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
39
- log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
40
- log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
41
-
42
- log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender
43
- log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH
44
- log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
45
- log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
46
- log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
47
-
48
- log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
49
- log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH
50
- log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
51
- log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
52
- log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
53
-
54
- log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender
55
- log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH
56
- log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
57
- log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
58
- log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
59
-
60
- # Change the line below to adjust ZK client logging
61
- log4j.logger.org.apache.zookeeper=INFO
62
-
63
- # Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
64
- log4j.logger.kafka=INFO
65
- log4j.logger.org.apache.kafka=INFO
66
-
67
- # Change to INFO or TRACE to enable request logging
68
- # log4j.logger.kafka.request.logger=TRACE, requestAppender
69
- # log4j.additivity.kafka.request.logger=false
70
-
71
- # Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output
72
- # related to the handling of requests
73
- # log4j.logger.kafka.network.Processor=TRACE, requestAppender
74
- log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
75
- # log4j.additivity.kafka.server.KafkaApis=false
76
- # log4j.logger.kafka.network.RequestChannel$=TRACE, requestAppender
77
- # log4j.additivity.kafka.network.RequestChannel$=false
78
-
79
- # Change the line below to adjust KRaft mode controller logging
80
- log4j.logger.org.apache.kafka.controller=INFO, controllerAppender
81
- log4j.additivity.org.apache.kafka.controller=false
82
-
83
- # Change the line below to adjust ZK mode controller logging
84
- log4j.logger.kafka.controller=INFO, controllerAppender
85
- log4j.additivity.kafka.controller=false
86
-
87
- log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
88
- log4j.additivity.kafka.log.LogCleaner=false
89
-
90
- log4j.logger.state.change.logger=INFO, stateChangeAppender
91
- log4j.additivity.state.change.logger=false
92
-
93
- # Access denials are logged at INFO level, change to INFO to also log allowed accesses
94
- log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender
95
- log4j.additivity.kafka.authorizer.logger=false
@@ -1,24 +0,0 @@
1
- #!/bin/bash
2
-
3
- # 1. Generating a x509 (CA) cert from a private key:
4
- openssl genrsa -out certs/ca.key 4096
5
- openssl req -new -x509 -key certs/ca.key -days 87660 -subj "/CN=kafka-ca" -out certs/ca.crt
6
-
7
- # 2. Generating a private key for kafka server and csr:
8
- openssl genrsa -out certs/kafka.key 4096
9
- openssl req -new -nodes -key certs/kafka.key -out certs/kafka.csr -subj "/CN=kafka"
10
- openssl x509 -req -in certs/kafka.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/kafka.crt -days 3650 -extensions SAN -extfile <(printf "[SAN]\nsubjectAltName=DNS:localhost")
11
-
12
- # 3. Generating keystore for kafka server:
13
- openssl pkcs12 -export -in certs/kafka.crt \
14
- -passout pass:password \
15
- -inkey certs/kafka.key \
16
- -out certs/kafka.keystore.jks
17
-
18
- # 4. Generating truststore for kafka server:
19
- keytool -importkeystore -srckeystore certs/kafka.keystore.jks \
20
- -srcstoretype PKCS12 \
21
- -srcstorepass password \
22
- -deststorepass password \
23
- -destkeystore certs/kafka.truststore.jks \
24
- -noprompt