kafka-ts 0.0.3-beta → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -8
- package/dist/api/api-versions.d.ts +9 -0
- package/{src/api/api-versions.ts → dist/api/api-versions.js} +8 -5
- package/dist/api/create-topics.d.ts +38 -0
- package/dist/api/create-topics.js +53 -0
- package/dist/api/delete-topics.d.ts +18 -0
- package/dist/api/delete-topics.js +33 -0
- package/dist/api/fetch.d.ts +84 -0
- package/dist/api/fetch.js +142 -0
- package/dist/api/find-coordinator.d.ts +21 -0
- package/{src/api/find-coordinator.ts → dist/api/find-coordinator.js} +14 -14
- package/dist/api/heartbeat.d.ts +11 -0
- package/dist/api/heartbeat.js +27 -0
- package/dist/api/index.d.ts +576 -0
- package/{src/api/index.ts → dist/api/index.js} +42 -41
- package/dist/api/init-producer-id.d.ts +13 -0
- package/dist/api/init-producer-id.js +29 -0
- package/dist/api/join-group.d.ts +34 -0
- package/dist/api/join-group.js +51 -0
- package/dist/api/leave-group.d.ts +19 -0
- package/dist/api/leave-group.js +39 -0
- package/dist/api/list-offsets.d.ts +29 -0
- package/dist/api/list-offsets.js +48 -0
- package/dist/api/metadata.d.ts +40 -0
- package/{src/api/metadata.ts → dist/api/metadata.js} +18 -26
- package/dist/api/offset-commit.d.ts +28 -0
- package/dist/api/offset-commit.js +48 -0
- package/dist/api/offset-fetch.d.ts +31 -0
- package/dist/api/offset-fetch.js +55 -0
- package/dist/api/produce.d.ts +54 -0
- package/{src/api/produce.ts → dist/api/produce.js} +55 -102
- package/dist/api/sasl-authenticate.d.ts +11 -0
- package/dist/api/sasl-authenticate.js +23 -0
- package/dist/api/sasl-handshake.d.ts +6 -0
- package/dist/api/sasl-handshake.js +19 -0
- package/dist/api/sync-group.d.ts +24 -0
- package/dist/api/sync-group.js +36 -0
- package/dist/auth/index.d.ts +2 -0
- package/dist/auth/index.js +8 -0
- package/dist/auth/plain.d.ts +5 -0
- package/dist/auth/plain.js +12 -0
- package/dist/auth/scram.d.ts +9 -0
- package/dist/auth/scram.js +40 -0
- package/dist/broker.d.ts +30 -0
- package/dist/broker.js +55 -0
- package/dist/client.d.ts +22 -0
- package/dist/client.js +36 -0
- package/dist/cluster.d.ts +27 -0
- package/dist/cluster.js +70 -0
- package/dist/cluster.test.d.ts +1 -0
- package/{src/cluster.test.ts → dist/cluster.test.js} +87 -113
- package/dist/codecs/gzip.d.ts +2 -0
- package/dist/codecs/gzip.js +8 -0
- package/dist/codecs/index.d.ts +2 -0
- package/dist/codecs/index.js +17 -0
- package/dist/codecs/none.d.ts +2 -0
- package/dist/codecs/none.js +7 -0
- package/dist/codecs/types.d.ts +5 -0
- package/dist/codecs/types.js +2 -0
- package/dist/connection.d.ts +26 -0
- package/dist/connection.js +175 -0
- package/dist/consumer/consumer-group.d.ts +41 -0
- package/dist/consumer/consumer-group.js +215 -0
- package/dist/consumer/consumer-metadata.d.ts +7 -0
- package/dist/consumer/consumer-metadata.js +14 -0
- package/dist/consumer/consumer.d.ts +44 -0
- package/dist/consumer/consumer.js +225 -0
- package/dist/consumer/fetch-manager.d.ts +33 -0
- package/dist/consumer/fetch-manager.js +140 -0
- package/dist/consumer/fetcher.d.ts +25 -0
- package/dist/consumer/fetcher.js +64 -0
- package/dist/consumer/offset-manager.d.ts +22 -0
- package/dist/consumer/offset-manager.js +66 -0
- package/dist/consumer/processor.d.ts +19 -0
- package/dist/consumer/processor.js +59 -0
- package/dist/distributors/assignments-to-replicas.d.ts +16 -0
- package/{src/distributors/assignments-to-replicas.ts → dist/distributors/assignments-to-replicas.js} +15 -41
- package/dist/distributors/assignments-to-replicas.test.d.ts +1 -0
- package/dist/distributors/assignments-to-replicas.test.js +40 -0
- package/dist/distributors/messages-to-topic-partition-leaders.d.ts +17 -0
- package/dist/distributors/messages-to-topic-partition-leaders.js +15 -0
- package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts +1 -0
- package/dist/distributors/messages-to-topic-partition-leaders.test.js +30 -0
- package/dist/distributors/partitioner.d.ts +7 -0
- package/dist/distributors/partitioner.js +23 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +26 -0
- package/dist/metadata.d.ts +24 -0
- package/dist/metadata.js +106 -0
- package/dist/producer/producer.d.ts +24 -0
- package/dist/producer/producer.js +131 -0
- package/{src/types.ts → dist/types.d.ts} +4 -4
- package/dist/types.js +2 -0
- package/{src/utils/api.ts → dist/utils/api.d.ts} +2 -4
- package/dist/utils/api.js +5 -0
- package/dist/utils/crypto.d.ts +8 -0
- package/dist/utils/crypto.js +18 -0
- package/dist/utils/decoder.d.ts +30 -0
- package/{src/utils/decoder.ts → dist/utils/decoder.js} +41 -57
- package/dist/utils/delay.d.ts +1 -0
- package/dist/utils/delay.js +5 -0
- package/dist/utils/encoder.d.ts +28 -0
- package/{src/utils/encoder.ts → dist/utils/encoder.js} +50 -66
- package/dist/utils/error.d.ts +11 -0
- package/dist/utils/error.js +27 -0
- package/dist/utils/logger.d.ts +9 -0
- package/dist/utils/logger.js +32 -0
- package/dist/utils/memo.d.ts +1 -0
- package/{src/utils/memo.ts → dist/utils/memo.js} +7 -3
- package/dist/utils/murmur2.d.ts +3 -0
- package/dist/utils/murmur2.js +40 -0
- package/dist/utils/retrier.d.ts +10 -0
- package/dist/utils/retrier.js +22 -0
- package/dist/utils/tracer.d.ts +5 -0
- package/dist/utils/tracer.js +39 -0
- package/package.json +11 -2
- package/.github/workflows/release.yml +0 -17
- package/.prettierrc +0 -8
- package/certs/ca.crt +0 -29
- package/certs/ca.key +0 -52
- package/certs/ca.srl +0 -1
- package/certs/kafka.crt +0 -29
- package/certs/kafka.csr +0 -26
- package/certs/kafka.key +0 -52
- package/certs/kafka.keystore.jks +0 -0
- package/certs/kafka.truststore.jks +0 -0
- package/docker-compose.yml +0 -104
- package/examples/package-lock.json +0 -31
- package/examples/package.json +0 -14
- package/examples/src/client.ts +0 -9
- package/examples/src/consumer.ts +0 -18
- package/examples/src/create-topic.ts +0 -44
- package/examples/src/producer.ts +0 -24
- package/examples/src/replicator.ts +0 -25
- package/examples/src/utils/delay.ts +0 -1
- package/examples/src/utils/json.ts +0 -1
- package/examples/tsconfig.json +0 -7
- package/log4j.properties +0 -95
- package/scripts/generate-certs.sh +0 -24
- package/src/__snapshots__/request-handler.test.ts.snap +0 -978
- package/src/api/create-topics.ts +0 -78
- package/src/api/delete-topics.ts +0 -42
- package/src/api/fetch.ts +0 -143
- package/src/api/heartbeat.ts +0 -33
- package/src/api/init-producer-id.ts +0 -35
- package/src/api/join-group.ts +0 -67
- package/src/api/leave-group.ts +0 -48
- package/src/api/list-offsets.ts +0 -65
- package/src/api/offset-commit.ts +0 -67
- package/src/api/offset-fetch.ts +0 -74
- package/src/api/sasl-authenticate.ts +0 -21
- package/src/api/sasl-handshake.ts +0 -16
- package/src/api/sync-group.ts +0 -54
- package/src/broker.ts +0 -74
- package/src/client.ts +0 -47
- package/src/cluster.ts +0 -87
- package/src/connection.ts +0 -143
- package/src/consumer/consumer-group.ts +0 -209
- package/src/consumer/consumer-metadata.ts +0 -14
- package/src/consumer/consumer.ts +0 -231
- package/src/consumer/fetch-manager.ts +0 -179
- package/src/consumer/fetcher.ts +0 -57
- package/src/consumer/offset-manager.ts +0 -93
- package/src/consumer/processor.ts +0 -47
- package/src/distributors/assignments-to-replicas.test.ts +0 -43
- package/src/distributors/messages-to-topic-partition-leaders.test.ts +0 -32
- package/src/distributors/messages-to-topic-partition-leaders.ts +0 -19
- package/src/index.ts +0 -4
- package/src/metadata.ts +0 -122
- package/src/producer/producer.ts +0 -132
- package/src/utils/debug.ts +0 -9
- package/src/utils/delay.ts +0 -1
- package/src/utils/error.ts +0 -21
- package/src/utils/retrier.ts +0 -39
- package/src/utils/tracer.ts +0 -31
- package/tsconfig.json +0 -17
package/certs/kafka.key
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
-----BEGIN PRIVATE KEY-----
|
|
2
|
-
MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDPMfQ6rqXKOj51
|
|
3
|
-
/MTiujAGpXgRzi82JdUKCa3Vl3ReDv2PceFpVl/fi0uJvEDUzipkgB7wTycuRIOC
|
|
4
|
-
8/AiHy6co/VoCf4/j97sTOHeV4kwV8+o//s5UApy3HC34o00ZwcP1yHJkN7Cnn1Z
|
|
5
|
-
cA/Txulv1RfNqjhWaVzsxcs5rsb8C3+Wn4NbttBo7YXMf1wDPHYNzRTk2agiE3vH
|
|
6
|
-
GqMrNAvrGmYmWhf+2a+LghG8EdBatXV0GnhHA+QhqWCq84CyzfOecFndq6niZx6V
|
|
7
|
-
3yj4yHDCjG+TefGV2SMuYgWSxAlV1g7lrzV63H910rhzLEjTbfET6n6LhCDZUCZX
|
|
8
|
-
U1JaOO+B186+7AUN75sVYHwKjHCFgivNmpnwGy4FiDfhs2V+3CoFGQEYDgU3PaYD
|
|
9
|
-
s0BFrDPnGeJtb/4bAaO8dzwaYjbb+rrks7YxzlHYJO51kAxCEZgU7OfL2vh3CdWK
|
|
10
|
-
7nBFy+rYjQGHM5hSI2pF+xQttRlA7qvYcPD+OIGy92D34QLlSnn97zZ9h+2lateK
|
|
11
|
-
bMOH915lQd1Ac280+ZEEwp2kEB3TFyH3e+iQnXwcaqmQGJsD4Kue3hn0sG6LksMM
|
|
12
|
-
zZstvVxAoWk9Mrp8s1+UG919Zc3p7/8ZovXtN+h1VG7NdKop4ZYretzyY50OY4vy
|
|
13
|
-
ao58GhPMO+O5n7d8I1YFTTfQzR2ZlwIDAQABAoICAACl6lqoozIfvoRF9yOya2YJ
|
|
14
|
-
7Hc5yuIDekgewaFyaNXGfDqPo6HduE0NtX3HzfgccPuBfxdHWBsH8dVeB/lDWbT5
|
|
15
|
-
ulHJ/jaar5rTnr1ATtiM3cRskI+cVXwIfn3EwqZQUDBXfu6PenLnZgL6m8IQ1Dga
|
|
16
|
-
VYm6J5T1nWnMMJeMmWi+bHa/Jv0JdFtvqsR5/YKxFLl0U8VnfOQmZXO4/Jus/QCO
|
|
17
|
-
Q5WJIJwWeDsafA23jIxMFwL29jtaosMMMZY3TsbAaCuttEpCODTSF8FmQk2M6vVg
|
|
18
|
-
1RSEBCJw3HBPtC7CWDkhg0hqpOMuf2OR9iu+DrhClsd2K1EqZ6PmYUhSpWZM2Urw
|
|
19
|
-
PCVaTl1ZSkogJVv2OmrpGWm3qfMJCOr1VYk5onOsANIZUeo9wYYZNcA7+ObR0mS2
|
|
20
|
-
tZrP+P5uP4FCVKA94+s/UOT5rkRw/jGJ8M776aOTXrrmJBMiUV1ExbmLbFmpX6FE
|
|
21
|
-
VInQAdsN1Shg8drUttHL3M1W5TXldAn/BBMiyFSbQu45EQqj+U+na/ilniHJeYzR
|
|
22
|
-
5vjJc61FA8ku1FQ4uYLe58xggVgVizl2/sdwPXVxUgbQ+YfnKtQNxvsY9fpk+dm6
|
|
23
|
-
x5bwqBCKiu6jQ0PN+/pxpc14UZmU8kvh+CZJ8G/jZzaovAINwxtS1pIzyXuTfLcT
|
|
24
|
-
WVBmgZpduYYEHKCg1DepAoIBAQD+vqddxTQmIgN8LH+3E/xBnrEsO3BHNulkMYwK
|
|
25
|
-
/RiPtokBAO0zWSHqhcClI+mqLX0wJmHSv5quXOWm37DBE0QAhFFXq+ZEDd2aPlXq
|
|
26
|
-
zU0g7Cmm3qhnPitGxNlvS3Gyf7OKuUQuvi2bRmKDxRHZ/idTn3OZaKAiMda7rZBR
|
|
27
|
-
ZT++zqRQ5dXjIO0ANIVgn2e4kW7CEo2dE0b7NNJ1RCrenk55MffOoVexWxDwbksM
|
|
28
|
-
Lt3kxpd3s7xCbVPTACJKX3aqxhq9sUDDHWQ/RwcJXnMb3N+V01W2fOjGcudHDPYe
|
|
29
|
-
OqAC1FaDF77hQpoeZZVy0fLfAJVWYEZV15oYkcVCx0idVSu1AoIBAQDQN1GvEhOp
|
|
30
|
-
dM3p8v98OWWkDLcvGPF8m/UdKdnQxazpLqye2gDOelBZjtlBJByqWCoYymEPIAbk
|
|
31
|
-
NzhsmiMNlQ+RI3rqGSDBqvagu3TpJp7DVWrPbzHyex/iSv18xxBiu4P3SVURwRcv
|
|
32
|
-
Nls4lBJnZ+MOcAtOupQ0XQke4+rcdTMq17OIwWkveYRNIk89HqAcqaEEbC3wAR3V
|
|
33
|
-
FMTj0As70nFdagbKWTfQRT+m4hCTa0u5H40Ej5U3ZNgV4kq91N8XRpf9i2FHpZcS
|
|
34
|
-
GZMSnizxjJpt1/M5z06wDN7kIdS6rjFn/vHBPVlCdxGvbX480BAi/AicmP4gqRiU
|
|
35
|
-
RTLCdJpZL3ebAoIBAF8hn6ra7CCZbye+Dmu2IOv4cT2NbN6LWRk7j4rC/ZsdlYg0
|
|
36
|
-
amBUlt/X1xtUj/wnRKroinuDZiZbNr/cpUtVAH4Y66+L2E/txaXGRZuL05PJ3bwd
|
|
37
|
-
RlJX2x1wgaqYR/tayRezHA1ApBNqc3u16na3/MOzth3B/psGdaRcqppa7zU4QhCb
|
|
38
|
-
5AYP07HR60UsAjXgybXb69qOymEhVr7IGCJOiAumZ15YzkDDO04uQgRotCCtMP9Y
|
|
39
|
-
taMw5K7VS/juMYiQtofEW+UBYuBmi/Apv17WWwC+jTG678q8OjZEVtzvE7Y6yZvn
|
|
40
|
-
EDdbAeb8FJVuZf51X0+EYH/KJheqyDmp5iorFkkCggEBAJLz2BTFn1bj90h8llBo
|
|
41
|
-
ehH0X31AD3+w2qhAl1g/i6/vnylmF1ei//6VUUQmZozbOKrSevof0n1Tc+FCUCcP
|
|
42
|
-
4zK+Zcu+/F1zeikq5HKBLWATcxUAF7JZr8EYyBs+q5PjYFoxSXcEXoaVyIAAEZUl
|
|
43
|
-
vxy7e1+PU3Y5i2xf06zQktJT9oQ4bTJygyulYWNzr/FKzze92hYf05xF0kgu1x7b
|
|
44
|
-
geM6J5y8pyW5GhmAR86j1YfMesyTQe8cn12inrmn+eQFkNQDgR7+AcdmgWSMeo/o
|
|
45
|
-
ej24QhO0+oZKLy23qtougvQw/RMzZ0H/o12623O0LQ9j7fgICqOmdJCA3jhb72uu
|
|
46
|
-
kq0CggEAfJceJjzSWN1wmr40DwAwgmHuBWh2y+0m/kyMGqRuPEIHTL2U8wCXIdmb
|
|
47
|
-
ugUjIbsb1lVdg3SdPnbV6X1CUuRoMd6XJ9cwR2NDgMr0DbCZsMq+g3dVKIPUFAoU
|
|
48
|
-
hB5Ai2fvrSKTdJsKHklSEoAP3Jf9GW8TXANiag8UhKYKN1Mc0kNx34LWiIjElneN
|
|
49
|
-
oRyE5pSnZcNeSkZFx2BYMd2YASHNbph8Nd9AIbV53CQ3yMD4mwDxnerkEvRsFKZd
|
|
50
|
-
FR7c8kID+YBN2D7I1HUN3uflt/VNZGPc1ZjFWk+UyXBf04TPRAG4C0qEshQC3nwA
|
|
51
|
-
0EaZY4cBQwWFep18Shu5QuXvsTbHmQ==
|
|
52
|
-
-----END PRIVATE KEY-----
|
package/certs/kafka.keystore.jks
DELETED
|
Binary file
|
|
Binary file
|
package/docker-compose.yml
DELETED
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
# kafka with raft:
|
|
2
|
-
services:
|
|
3
|
-
kafka-0:
|
|
4
|
-
container_name: kafka-0
|
|
5
|
-
image: apache/kafka:3.7.1
|
|
6
|
-
ports:
|
|
7
|
-
- '9092:9092'
|
|
8
|
-
- '29092:29092'
|
|
9
|
-
environment:
|
|
10
|
-
KAFKA_NODE_ID: 0
|
|
11
|
-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
|
|
12
|
-
KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
|
|
13
|
-
KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
|
|
14
|
-
KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092,INTERBROKER://kafka-0:19092'
|
|
15
|
-
KAFKA_PROCESS_ROLES: 'broker,controller'
|
|
16
|
-
KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
|
|
17
|
-
KAFKA_LISTENERS: 'EXTERNAL://:9092,INTERBROKER://:19092,CONTROLLER://:29092'
|
|
18
|
-
KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
|
|
19
|
-
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
|
|
20
|
-
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
|
|
21
|
-
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
22
|
-
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
|
|
23
|
-
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
|
24
|
-
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
|
25
|
-
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
|
|
26
|
-
KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
|
|
27
|
-
KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
|
|
28
|
-
KAFKA_SSL_KEY_PASSWORD: 'password'
|
|
29
|
-
KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
|
|
30
|
-
KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
|
|
31
|
-
KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
|
|
32
|
-
KAFKA_SSL_CLIENT_AUTH: 'required'
|
|
33
|
-
volumes:
|
|
34
|
-
- ./log4j.properties:/etc/kafka/docker/log4j.properties
|
|
35
|
-
- ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
|
|
36
|
-
- ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
|
|
37
|
-
kafka-1:
|
|
38
|
-
container_name: kafka-1
|
|
39
|
-
image: apache/kafka:3.7.1
|
|
40
|
-
ports:
|
|
41
|
-
- '9093:9093'
|
|
42
|
-
- '29093:29093'
|
|
43
|
-
environment:
|
|
44
|
-
KAFKA_NODE_ID: 1
|
|
45
|
-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
|
|
46
|
-
KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
|
|
47
|
-
KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
|
|
48
|
-
KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9093,INTERBROKER://kafka-1:19093'
|
|
49
|
-
KAFKA_PROCESS_ROLES: 'broker,controller'
|
|
50
|
-
KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
|
|
51
|
-
KAFKA_LISTENERS: 'EXTERNAL://:9093,INTERBROKER://:19093,CONTROLLER://:29093'
|
|
52
|
-
KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
|
|
53
|
-
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
|
|
54
|
-
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
|
|
55
|
-
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
56
|
-
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
|
|
57
|
-
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
|
58
|
-
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
|
59
|
-
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
|
|
60
|
-
KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
|
|
61
|
-
KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
|
|
62
|
-
KAFKA_SSL_KEY_PASSWORD: 'password'
|
|
63
|
-
KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
|
|
64
|
-
KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
|
|
65
|
-
KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
|
|
66
|
-
KAFKA_SSL_CLIENT_AUTH: 'required'
|
|
67
|
-
volumes:
|
|
68
|
-
- ./log4j.properties:/etc/kafka/docker/log4j.properties
|
|
69
|
-
- ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
|
|
70
|
-
- ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
|
|
71
|
-
kafka-2:
|
|
72
|
-
container_name: kafka-2
|
|
73
|
-
image: apache/kafka:3.7.1
|
|
74
|
-
ports:
|
|
75
|
-
- '9094:9094'
|
|
76
|
-
- '29094:29094'
|
|
77
|
-
environment:
|
|
78
|
-
KAFKA_NODE_ID: 2
|
|
79
|
-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERBROKER:PLAINTEXT,EXTERNAL:SASL_SSL'
|
|
80
|
-
KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin";'
|
|
81
|
-
KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN'
|
|
82
|
-
KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9094,INTERBROKER://kafka-2:19094'
|
|
83
|
-
KAFKA_PROCESS_ROLES: 'broker,controller'
|
|
84
|
-
KAFKA_CONTROLLER_QUORUM_VOTERS: '0@kafka-0:29092,1@kafka-1:29093,2@kafka-2:29094'
|
|
85
|
-
KAFKA_LISTENERS: 'EXTERNAL://:9094,INTERBROKER://:19094,CONTROLLER://:29094'
|
|
86
|
-
KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERBROKER'
|
|
87
|
-
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
|
|
88
|
-
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
|
|
89
|
-
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
90
|
-
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
|
|
91
|
-
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
|
92
|
-
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
|
93
|
-
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
|
|
94
|
-
KAFKA_SSL_KEYSTORE_LOCATION: '/run/secrets/kafka.server.keystore.jks'
|
|
95
|
-
KAFKA_SSL_KEYSTORE_PASSWORD: 'password'
|
|
96
|
-
KAFKA_SSL_KEY_PASSWORD: 'password'
|
|
97
|
-
KAFKA_SSL_TRUSTSTORE_LOCATION: '/run/secrets/kafka.server.truststore.jks'
|
|
98
|
-
KAFKA_SSL_TRUSTSTORE_PASSWORD: 'password'
|
|
99
|
-
KAFKA_SSL_ENDPOINT_IDENTIFICATION.algorithm: ''
|
|
100
|
-
KAFKA_SSL_CLIENT_AUTH: 'required'
|
|
101
|
-
volumes:
|
|
102
|
-
- ./log4j.properties:/etc/kafka/docker/log4j.properties
|
|
103
|
-
- ./certs/kafka.truststore.jks:/run/secrets/kafka.server.truststore.jks
|
|
104
|
-
- ./certs/kafka.keystore.jks:/run/secrets/kafka.server.keystore.jks
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "examples",
|
|
3
|
-
"version": "1.0.0",
|
|
4
|
-
"lockfileVersion": 3,
|
|
5
|
-
"requires": true,
|
|
6
|
-
"packages": {
|
|
7
|
-
"": {
|
|
8
|
-
"name": "examples",
|
|
9
|
-
"version": "1.0.0",
|
|
10
|
-
"license": "ISC",
|
|
11
|
-
"dependencies": {
|
|
12
|
-
"kafka-ts": "file:../"
|
|
13
|
-
}
|
|
14
|
-
},
|
|
15
|
-
"..": {
|
|
16
|
-
"name": "kafka-ts",
|
|
17
|
-
"version": "0.0.1-beta",
|
|
18
|
-
"license": "MIT",
|
|
19
|
-
"devDependencies": {
|
|
20
|
-
"@types/node": "^20.12.12",
|
|
21
|
-
"prettier": "^3.2.5",
|
|
22
|
-
"typescript": "^5.4.5",
|
|
23
|
-
"vitest": "^1.6.0"
|
|
24
|
-
}
|
|
25
|
-
},
|
|
26
|
-
"node_modules/kafka-ts": {
|
|
27
|
-
"resolved": "..",
|
|
28
|
-
"link": true
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
}
|
package/examples/package.json
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "examples",
|
|
3
|
-
"version": "1.0.0",
|
|
4
|
-
"description": "",
|
|
5
|
-
"main": "dist/replicator.js",
|
|
6
|
-
"scripts": {
|
|
7
|
-
"test": "echo \"Error: no test specified\" && exit 1"
|
|
8
|
-
},
|
|
9
|
-
"dependencies": {
|
|
10
|
-
"kafka-ts": "file:../"
|
|
11
|
-
},
|
|
12
|
-
"author": "",
|
|
13
|
-
"license": "ISC"
|
|
14
|
-
}
|
package/examples/src/client.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { readFileSync } from 'fs';
|
|
2
|
-
import { createKafkaClient } from 'kafka-ts';
|
|
3
|
-
|
|
4
|
-
export const kafka = createKafkaClient({
|
|
5
|
-
clientId: 'examples',
|
|
6
|
-
bootstrapServers: [{ host: 'localhost', port: 9092 }],
|
|
7
|
-
sasl: { mechanism: 'PLAIN', username: 'admin', password: 'admin' },
|
|
8
|
-
ssl: { ca: readFileSync('../certs/ca.crt').toString() },
|
|
9
|
-
});
|
package/examples/src/consumer.ts
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { kafka } from './client';
|
|
2
|
-
|
|
3
|
-
(async () => {
|
|
4
|
-
const consumer = await kafka.startConsumer({
|
|
5
|
-
groupId: 'example-group',
|
|
6
|
-
groupInstanceId: 'example-group-instance',
|
|
7
|
-
topics: ['my-topic'],
|
|
8
|
-
onBatch: (batch) => {
|
|
9
|
-
console.log(batch);
|
|
10
|
-
},
|
|
11
|
-
granularity: 'broker',
|
|
12
|
-
concurrency: 10,
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
process.on('SIGINT', async () => {
|
|
16
|
-
await consumer.close();
|
|
17
|
-
});
|
|
18
|
-
})();
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
import { API, API_ERROR, KafkaTSApiError } from 'kafka-ts';
|
|
2
|
-
import { kafka } from './client';
|
|
3
|
-
|
|
4
|
-
(async () => {
|
|
5
|
-
const cluster = kafka.createCluster();
|
|
6
|
-
await cluster.connect();
|
|
7
|
-
|
|
8
|
-
const { controllerId } = await cluster.sendRequest(API.METADATA, {
|
|
9
|
-
allowTopicAutoCreation: false,
|
|
10
|
-
includeTopicAuthorizedOperations: false,
|
|
11
|
-
topics: [],
|
|
12
|
-
});
|
|
13
|
-
|
|
14
|
-
try {
|
|
15
|
-
await cluster.sendRequestToNode(controllerId)(API.CREATE_TOPICS, {
|
|
16
|
-
validateOnly: false,
|
|
17
|
-
timeoutMs: 10_000,
|
|
18
|
-
topics: [
|
|
19
|
-
{
|
|
20
|
-
name: 'my-topic',
|
|
21
|
-
numPartitions: 10,
|
|
22
|
-
replicationFactor: 3,
|
|
23
|
-
assignments: [],
|
|
24
|
-
configs: [],
|
|
25
|
-
},
|
|
26
|
-
],
|
|
27
|
-
});
|
|
28
|
-
} catch (error) {
|
|
29
|
-
if ((error as KafkaTSApiError).errorCode === API_ERROR.TOPIC_ALREADY_EXISTS) {
|
|
30
|
-
return;
|
|
31
|
-
}
|
|
32
|
-
throw error;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
const metadata = await cluster.sendRequestToNode(controllerId)(API.METADATA, {
|
|
36
|
-
allowTopicAutoCreation: false,
|
|
37
|
-
includeTopicAuthorizedOperations: false,
|
|
38
|
-
topics: [{ id: null, name: 'my-topic' }],
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
console.log(metadata);
|
|
42
|
-
|
|
43
|
-
await cluster.disconnect();
|
|
44
|
-
})();
|
package/examples/src/producer.ts
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { createInterface } from 'readline';
|
|
2
|
-
import { kafka } from './client';
|
|
3
|
-
|
|
4
|
-
const producer = kafka.createProducer({ allowTopicAutoCreation: true });
|
|
5
|
-
|
|
6
|
-
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
7
|
-
|
|
8
|
-
process.stdout.write('> ');
|
|
9
|
-
rl.on('line', async (line) => {
|
|
10
|
-
await producer.send([
|
|
11
|
-
{
|
|
12
|
-
topic: 'example-topic-f',
|
|
13
|
-
key: null,
|
|
14
|
-
value: line,
|
|
15
|
-
partition: 0,
|
|
16
|
-
},
|
|
17
|
-
]);
|
|
18
|
-
process.stdout.write('> ');
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
process.on('SIGINT', async () => {
|
|
22
|
-
rl.close();
|
|
23
|
-
await producer.close();
|
|
24
|
-
});
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
import { kafka } from './client';
|
|
2
|
-
|
|
3
|
-
(async () => {
|
|
4
|
-
const topic = 'example-topic';
|
|
5
|
-
|
|
6
|
-
const producer = kafka.createProducer({ allowTopicAutoCreation: true });
|
|
7
|
-
const consumer = await kafka.startConsumer({
|
|
8
|
-
topics: [topic],
|
|
9
|
-
onBatch: async (messages) => {
|
|
10
|
-
await producer.send(
|
|
11
|
-
messages.map((message) => ({
|
|
12
|
-
...message,
|
|
13
|
-
headers: { 'X-Replicated': 'true' },
|
|
14
|
-
topic: `${message.topic}-replicated`,
|
|
15
|
-
offset: 0n,
|
|
16
|
-
})),
|
|
17
|
-
);
|
|
18
|
-
console.log(`Replicated ${messages.length} messages`);
|
|
19
|
-
},
|
|
20
|
-
});
|
|
21
|
-
process.on('SIGINT', async () => {
|
|
22
|
-
await consumer.close();
|
|
23
|
-
await producer.close();
|
|
24
|
-
});
|
|
25
|
-
})();
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export const delay = (delayMs: number) => new Promise<void>((resolve) => setTimeout(resolve, delayMs));
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export const serializer = (_: string, value: unknown) => (typeof value === 'bigint' ? value.toString() : value);
|
package/examples/tsconfig.json
DELETED
package/log4j.properties
DELETED
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
2
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
3
|
-
# this work for additional information regarding copyright ownership.
|
|
4
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
5
|
-
# (the "License"); you may not use this file except in compliance with
|
|
6
|
-
# the License. You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
|
|
16
|
-
# Unspecified loggers and loggers with additivity=true output to server.log and stdout
|
|
17
|
-
# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
|
|
18
|
-
log4j.rootLogger=INFO, stdout, kafkaAppender
|
|
19
|
-
|
|
20
|
-
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
|
21
|
-
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
|
22
|
-
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
|
|
23
|
-
|
|
24
|
-
log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
|
|
25
|
-
log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH
|
|
26
|
-
log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
|
|
27
|
-
log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
|
|
28
|
-
log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
|
29
|
-
|
|
30
|
-
log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender
|
|
31
|
-
log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH
|
|
32
|
-
log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
|
|
33
|
-
log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
|
|
34
|
-
log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
|
35
|
-
|
|
36
|
-
log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender
|
|
37
|
-
log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH
|
|
38
|
-
log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
|
|
39
|
-
log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
|
|
40
|
-
log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
|
41
|
-
|
|
42
|
-
log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender
|
|
43
|
-
log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH
|
|
44
|
-
log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
|
|
45
|
-
log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
|
|
46
|
-
log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
|
47
|
-
|
|
48
|
-
log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
|
|
49
|
-
log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH
|
|
50
|
-
log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
|
|
51
|
-
log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
|
|
52
|
-
log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
|
53
|
-
|
|
54
|
-
log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender
|
|
55
|
-
log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH
|
|
56
|
-
log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
|
|
57
|
-
log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
|
|
58
|
-
log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
|
|
59
|
-
|
|
60
|
-
# Change the line below to adjust ZK client logging
|
|
61
|
-
log4j.logger.org.apache.zookeeper=INFO
|
|
62
|
-
|
|
63
|
-
# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
|
|
64
|
-
log4j.logger.kafka=INFO
|
|
65
|
-
log4j.logger.org.apache.kafka=INFO
|
|
66
|
-
|
|
67
|
-
# Change to INFO or TRACE to enable request logging
|
|
68
|
-
# log4j.logger.kafka.request.logger=TRACE, requestAppender
|
|
69
|
-
# log4j.additivity.kafka.request.logger=false
|
|
70
|
-
|
|
71
|
-
# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output
|
|
72
|
-
# related to the handling of requests
|
|
73
|
-
# log4j.logger.kafka.network.Processor=TRACE, requestAppender
|
|
74
|
-
log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
|
|
75
|
-
# log4j.additivity.kafka.server.KafkaApis=false
|
|
76
|
-
# log4j.logger.kafka.network.RequestChannel$=TRACE, requestAppender
|
|
77
|
-
# log4j.additivity.kafka.network.RequestChannel$=false
|
|
78
|
-
|
|
79
|
-
# Change the line below to adjust KRaft mode controller logging
|
|
80
|
-
log4j.logger.org.apache.kafka.controller=INFO, controllerAppender
|
|
81
|
-
log4j.additivity.org.apache.kafka.controller=false
|
|
82
|
-
|
|
83
|
-
# Change the line below to adjust ZK mode controller logging
|
|
84
|
-
log4j.logger.kafka.controller=INFO, controllerAppender
|
|
85
|
-
log4j.additivity.kafka.controller=false
|
|
86
|
-
|
|
87
|
-
log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
|
|
88
|
-
log4j.additivity.kafka.log.LogCleaner=false
|
|
89
|
-
|
|
90
|
-
log4j.logger.state.change.logger=INFO, stateChangeAppender
|
|
91
|
-
log4j.additivity.state.change.logger=false
|
|
92
|
-
|
|
93
|
-
# Access denials are logged at INFO level, change to INFO to also log allowed accesses
|
|
94
|
-
log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender
|
|
95
|
-
log4j.additivity.kafka.authorizer.logger=false
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
|
|
3
|
-
# 1. Generating a x509 (CA) cert from a private key:
|
|
4
|
-
openssl genrsa -out certs/ca.key 4096
|
|
5
|
-
openssl req -new -x509 -key certs/ca.key -days 87660 -subj "/CN=kafka-ca" -out certs/ca.crt
|
|
6
|
-
|
|
7
|
-
# 2. Generating a private key for kafka server and csr:
|
|
8
|
-
openssl genrsa -out certs/kafka.key 4096
|
|
9
|
-
openssl req -new -nodes -key certs/kafka.key -out certs/kafka.csr -subj "/CN=kafka"
|
|
10
|
-
openssl x509 -req -in certs/kafka.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/kafka.crt -days 3650 -extensions SAN -extfile <(printf "[SAN]\nsubjectAltName=DNS:localhost")
|
|
11
|
-
|
|
12
|
-
# 3. Generating keystore for kafka server:
|
|
13
|
-
openssl pkcs12 -export -in certs/kafka.crt \
|
|
14
|
-
-passout pass:password \
|
|
15
|
-
-inkey certs/kafka.key \
|
|
16
|
-
-out certs/kafka.keystore.jks
|
|
17
|
-
|
|
18
|
-
# 4. Generating truststore for kafka server:
|
|
19
|
-
keytool -importkeystore -srckeystore certs/kafka.keystore.jks \
|
|
20
|
-
-srcstoretype PKCS12 \
|
|
21
|
-
-srcstorepass password \
|
|
22
|
-
-deststorepass password \
|
|
23
|
-
-destkeystore certs/kafka.truststore.jks \
|
|
24
|
-
-noprompt
|