kafka-ts 0.0.3-beta → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -8
- package/dist/api/api-versions.d.ts +9 -0
- package/dist/api/api-versions.js +24 -0
- package/dist/api/create-topics.d.ts +38 -0
- package/dist/api/create-topics.js +53 -0
- package/dist/api/delete-topics.d.ts +18 -0
- package/dist/api/delete-topics.js +33 -0
- package/dist/api/fetch.d.ts +84 -0
- package/dist/api/fetch.js +142 -0
- package/dist/api/find-coordinator.d.ts +21 -0
- package/dist/api/find-coordinator.js +39 -0
- package/dist/api/heartbeat.d.ts +11 -0
- package/dist/api/heartbeat.js +27 -0
- package/dist/api/index.d.ts +576 -0
- package/dist/api/index.js +165 -0
- package/dist/api/init-producer-id.d.ts +13 -0
- package/dist/api/init-producer-id.js +29 -0
- package/dist/api/join-group.d.ts +34 -0
- package/dist/api/join-group.js +51 -0
- package/dist/api/leave-group.d.ts +19 -0
- package/dist/api/leave-group.js +39 -0
- package/dist/api/list-offsets.d.ts +29 -0
- package/dist/api/list-offsets.js +48 -0
- package/dist/api/metadata.d.ts +40 -0
- package/dist/api/metadata.js +58 -0
- package/dist/api/offset-commit.d.ts +28 -0
- package/dist/api/offset-commit.js +48 -0
- package/dist/api/offset-fetch.d.ts +31 -0
- package/dist/api/offset-fetch.js +55 -0
- package/dist/api/produce.d.ts +54 -0
- package/dist/api/produce.js +126 -0
- package/dist/api/sasl-authenticate.d.ts +11 -0
- package/dist/api/sasl-authenticate.js +23 -0
- package/dist/api/sasl-handshake.d.ts +6 -0
- package/dist/api/sasl-handshake.js +19 -0
- package/dist/api/sync-group.d.ts +24 -0
- package/dist/api/sync-group.js +36 -0
- package/dist/auth/index.d.ts +2 -0
- package/dist/auth/index.js +8 -0
- package/dist/auth/plain.d.ts +5 -0
- package/dist/auth/plain.js +12 -0
- package/dist/auth/scram.d.ts +9 -0
- package/dist/auth/scram.js +40 -0
- package/dist/broker.d.ts +30 -0
- package/dist/broker.js +55 -0
- package/dist/client.d.ts +23 -0
- package/dist/client.js +36 -0
- package/dist/cluster.d.ts +27 -0
- package/dist/cluster.js +70 -0
- package/dist/cluster.test.d.ts +1 -0
- package/dist/cluster.test.js +343 -0
- package/dist/codecs/gzip.d.ts +2 -0
- package/dist/codecs/gzip.js +8 -0
- package/dist/codecs/index.d.ts +2 -0
- package/dist/codecs/index.js +17 -0
- package/dist/codecs/none.d.ts +2 -0
- package/dist/codecs/none.js +7 -0
- package/dist/codecs/types.d.ts +5 -0
- package/dist/codecs/types.js +2 -0
- package/dist/connection.d.ts +26 -0
- package/dist/connection.js +175 -0
- package/dist/consumer/consumer-group.d.ts +41 -0
- package/dist/consumer/consumer-group.js +215 -0
- package/dist/consumer/consumer-metadata.d.ts +7 -0
- package/dist/consumer/consumer-metadata.js +14 -0
- package/dist/consumer/consumer.d.ts +44 -0
- package/dist/consumer/consumer.js +225 -0
- package/dist/consumer/fetch-manager.d.ts +33 -0
- package/dist/consumer/fetch-manager.js +140 -0
- package/dist/consumer/fetcher.d.ts +25 -0
- package/dist/consumer/fetcher.js +64 -0
- package/dist/consumer/offset-manager.d.ts +22 -0
- package/dist/consumer/offset-manager.js +66 -0
- package/dist/consumer/processor.d.ts +19 -0
- package/dist/consumer/processor.js +59 -0
- package/dist/distributors/assignments-to-replicas.d.ts +16 -0
- package/dist/distributors/assignments-to-replicas.js +59 -0
- package/dist/distributors/assignments-to-replicas.test.d.ts +1 -0
- package/dist/distributors/assignments-to-replicas.test.js +40 -0
- package/dist/distributors/messages-to-topic-partition-leaders.d.ts +17 -0
- package/dist/distributors/messages-to-topic-partition-leaders.js +15 -0
- package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts +1 -0
- package/dist/distributors/messages-to-topic-partition-leaders.test.js +30 -0
- package/dist/distributors/partitioner.d.ts +7 -0
- package/dist/distributors/partitioner.js +23 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +26 -0
- package/dist/metadata.d.ts +24 -0
- package/dist/metadata.js +106 -0
- package/dist/producer/producer.d.ts +24 -0
- package/dist/producer/producer.js +131 -0
- package/dist/types.d.ts +11 -0
- package/dist/types.js +2 -0
- package/dist/utils/api.d.ts +9 -0
- package/dist/utils/api.js +5 -0
- package/dist/utils/crypto.d.ts +8 -0
- package/dist/utils/crypto.js +18 -0
- package/dist/utils/decoder.d.ts +30 -0
- package/dist/utils/decoder.js +152 -0
- package/dist/utils/delay.d.ts +1 -0
- package/dist/utils/delay.js +5 -0
- package/dist/utils/encoder.d.ts +28 -0
- package/dist/utils/encoder.js +125 -0
- package/dist/utils/error.d.ts +11 -0
- package/dist/utils/error.js +27 -0
- package/dist/utils/logger.d.ts +9 -0
- package/dist/utils/logger.js +32 -0
- package/dist/utils/memo.d.ts +1 -0
- package/dist/utils/memo.js +16 -0
- package/dist/utils/murmur2.d.ts +3 -0
- package/dist/utils/murmur2.js +40 -0
- package/dist/utils/retrier.d.ts +10 -0
- package/dist/utils/retrier.js +22 -0
- package/dist/utils/tracer.d.ts +5 -0
- package/dist/utils/tracer.js +39 -0
- package/package.json +11 -2
- package/src/__snapshots__/{request-handler.test.ts.snap → cluster.test.ts.snap} +329 -26
- package/src/api/fetch.ts +84 -29
- package/src/api/index.ts +3 -1
- package/src/api/metadata.ts +1 -1
- package/src/api/offset-commit.ts +1 -1
- package/src/api/offset-fetch.ts +1 -5
- package/src/api/produce.ts +15 -18
- package/src/auth/index.ts +2 -0
- package/src/auth/plain.ts +10 -0
- package/src/auth/scram.ts +52 -0
- package/src/broker.ts +7 -9
- package/src/client.ts +2 -2
- package/src/cluster.test.ts +16 -14
- package/src/cluster.ts +38 -40
- package/src/codecs/gzip.ts +9 -0
- package/src/codecs/index.ts +16 -0
- package/src/codecs/none.ts +6 -0
- package/src/codecs/types.ts +4 -0
- package/src/connection.ts +31 -17
- package/src/consumer/consumer-group.ts +43 -23
- package/src/consumer/consumer.ts +64 -43
- package/src/consumer/fetch-manager.ts +43 -53
- package/src/consumer/fetcher.ts +20 -13
- package/src/consumer/offset-manager.ts +18 -7
- package/src/consumer/processor.ts +14 -8
- package/src/distributors/assignments-to-replicas.ts +1 -3
- package/src/distributors/partitioner.ts +27 -0
- package/src/index.ts +7 -2
- package/src/metadata.ts +4 -0
- package/src/producer/producer.ts +22 -12
- package/src/types.ts +3 -3
- package/src/utils/api.ts +1 -1
- package/src/utils/crypto.ts +15 -0
- package/src/utils/decoder.ts +11 -5
- package/src/utils/encoder.ts +29 -22
- package/src/utils/logger.ts +37 -0
- package/src/utils/murmur2.ts +44 -0
- package/src/utils/tracer.ts +40 -22
- package/.github/workflows/release.yml +0 -17
- package/certs/ca.crt +0 -29
- package/certs/ca.key +0 -52
- package/certs/ca.srl +0 -1
- package/certs/kafka.crt +0 -29
- package/certs/kafka.csr +0 -26
- package/certs/kafka.key +0 -52
- package/certs/kafka.keystore.jks +0 -0
- package/certs/kafka.truststore.jks +0 -0
- package/docker-compose.yml +0 -104
- package/examples/package-lock.json +0 -31
- package/examples/package.json +0 -14
- package/examples/src/client.ts +0 -9
- package/examples/src/consumer.ts +0 -18
- package/examples/src/create-topic.ts +0 -44
- package/examples/src/producer.ts +0 -24
- package/examples/src/replicator.ts +0 -25
- package/examples/src/utils/delay.ts +0 -1
- package/examples/src/utils/json.ts +0 -1
- package/examples/tsconfig.json +0 -7
- package/log4j.properties +0 -95
- package/scripts/generate-certs.sh +0 -24
- package/src/utils/debug.ts +0 -9
package/dist/cluster.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Cluster = void 0;
|
|
4
|
+
const api_1 = require("./api");
|
|
5
|
+
const broker_1 = require("./broker");
|
|
6
|
+
const error_1 = require("./utils/error");
|
|
7
|
+
const logger_1 = require("./utils/logger");
|
|
8
|
+
class Cluster {
|
|
9
|
+
options;
|
|
10
|
+
seedBroker = new broker_1.Broker({ clientId: null, sasl: null, ssl: null, options: { port: 9092 } });
|
|
11
|
+
brokerById = {};
|
|
12
|
+
brokerMetadata = {};
|
|
13
|
+
constructor(options) {
|
|
14
|
+
this.options = options;
|
|
15
|
+
}
|
|
16
|
+
async connect() {
|
|
17
|
+
this.seedBroker = await this.findSeedBroker();
|
|
18
|
+
this.brokerById = {};
|
|
19
|
+
const metadata = await this.sendRequest(api_1.API.METADATA, {
|
|
20
|
+
allowTopicAutoCreation: false,
|
|
21
|
+
includeTopicAuthorizedOperations: false,
|
|
22
|
+
topics: [],
|
|
23
|
+
});
|
|
24
|
+
this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
|
|
25
|
+
}
|
|
26
|
+
async disconnect() {
|
|
27
|
+
await Promise.all([this.seedBroker.disconnect(), ...Object.values(this.brokerById).map((x) => x.disconnect())]);
|
|
28
|
+
}
|
|
29
|
+
setSeedBroker = async (nodeId) => {
|
|
30
|
+
await this.seedBroker.disconnect();
|
|
31
|
+
this.seedBroker = await this.acquireBroker(nodeId);
|
|
32
|
+
};
|
|
33
|
+
sendRequest = (...args) => this.seedBroker.sendRequest(...args);
|
|
34
|
+
sendRequestToNode = (nodeId) => async (...args) => {
|
|
35
|
+
if (!this.brokerById[nodeId]) {
|
|
36
|
+
this.brokerById[nodeId] = await this.acquireBroker(nodeId);
|
|
37
|
+
}
|
|
38
|
+
return this.brokerById[nodeId].sendRequest(...args);
|
|
39
|
+
};
|
|
40
|
+
async acquireBroker(nodeId) {
|
|
41
|
+
const broker = new broker_1.Broker({
|
|
42
|
+
clientId: this.options.clientId,
|
|
43
|
+
sasl: this.options.sasl,
|
|
44
|
+
ssl: this.options.ssl,
|
|
45
|
+
options: this.brokerMetadata[nodeId],
|
|
46
|
+
});
|
|
47
|
+
await broker.connect();
|
|
48
|
+
return broker;
|
|
49
|
+
}
|
|
50
|
+
async findSeedBroker() {
|
|
51
|
+
const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
|
|
52
|
+
for (const options of randomizedBrokers) {
|
|
53
|
+
try {
|
|
54
|
+
const broker = await new broker_1.Broker({
|
|
55
|
+
clientId: this.options.clientId,
|
|
56
|
+
sasl: this.options.sasl,
|
|
57
|
+
ssl: this.options.ssl,
|
|
58
|
+
options,
|
|
59
|
+
});
|
|
60
|
+
await broker.connect();
|
|
61
|
+
return broker;
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
logger_1.log.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
throw new error_1.KafkaTSError('No seed brokers found');
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
exports.Cluster = Cluster;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const crypto_1 = require("crypto");
|
|
4
|
+
const fs_1 = require("fs");
|
|
5
|
+
const vitest_1 = require("vitest");
|
|
6
|
+
const api_1 = require("./api");
|
|
7
|
+
const find_coordinator_1 = require("./api/find-coordinator");
|
|
8
|
+
const auth_1 = require("./auth");
|
|
9
|
+
const client_1 = require("./client");
|
|
10
|
+
const kafka = (0, client_1.createKafkaClient)({
|
|
11
|
+
clientId: 'kafka-ts',
|
|
12
|
+
bootstrapServers: [{ host: 'localhost', port: 9092 }],
|
|
13
|
+
sasl: (0, auth_1.saslPlain)({ username: 'admin', password: 'admin' }),
|
|
14
|
+
ssl: { ca: (0, fs_1.readFileSync)('./certs/ca.crt').toString() },
|
|
15
|
+
});
|
|
16
|
+
vitest_1.describe.sequential('Low-level API', () => {
|
|
17
|
+
const groupId = (0, crypto_1.randomBytes)(16).toString('hex');
|
|
18
|
+
let cluster;
|
|
19
|
+
(0, vitest_1.beforeAll)(async () => {
|
|
20
|
+
cluster = await kafka.createCluster();
|
|
21
|
+
await cluster.connect();
|
|
22
|
+
const metadataResult = await cluster.sendRequest(api_1.API.METADATA, {
|
|
23
|
+
topics: null,
|
|
24
|
+
allowTopicAutoCreation: false,
|
|
25
|
+
includeTopicAuthorizedOperations: false,
|
|
26
|
+
});
|
|
27
|
+
if (metadataResult.topics.some((topic) => topic.name === 'kafka-ts-test-topic')) {
|
|
28
|
+
await cluster.sendRequest(api_1.API.DELETE_TOPICS, {
|
|
29
|
+
topics: [{ name: 'kafka-ts-test-topic', topicId: null }],
|
|
30
|
+
timeoutMs: 10000,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
});
|
|
34
|
+
(0, vitest_1.afterAll)(async () => {
|
|
35
|
+
await cluster.disconnect();
|
|
36
|
+
});
|
|
37
|
+
(0, vitest_1.it)('should request api versions', async () => {
|
|
38
|
+
const result = await cluster.sendRequest(api_1.API.API_VERSIONS, {});
|
|
39
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
40
|
+
});
|
|
41
|
+
let topicId = 'd6718d178e1b47c886441ad2d19faea5';
|
|
42
|
+
(0, vitest_1.it)('should create topics', async () => {
|
|
43
|
+
const result = await cluster.sendRequest(api_1.API.CREATE_TOPICS, {
|
|
44
|
+
topics: [
|
|
45
|
+
{
|
|
46
|
+
name: 'kafka-ts-test-topic',
|
|
47
|
+
numPartitions: 10,
|
|
48
|
+
replicationFactor: 3,
|
|
49
|
+
assignments: [],
|
|
50
|
+
configs: [],
|
|
51
|
+
},
|
|
52
|
+
],
|
|
53
|
+
timeoutMs: 10000,
|
|
54
|
+
validateOnly: false,
|
|
55
|
+
});
|
|
56
|
+
topicId = result.topics[0].topicId;
|
|
57
|
+
result.topics.forEach((topic) => {
|
|
58
|
+
topic.topicId = 'Any<UUID>';
|
|
59
|
+
});
|
|
60
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
61
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
62
|
+
});
|
|
63
|
+
(0, vitest_1.it)('should request metadata for all topics', async () => {
|
|
64
|
+
const result = await cluster.sendRequest(api_1.API.METADATA, {
|
|
65
|
+
topics: null,
|
|
66
|
+
allowTopicAutoCreation: false,
|
|
67
|
+
includeTopicAuthorizedOperations: false,
|
|
68
|
+
});
|
|
69
|
+
result.controllerId = 0;
|
|
70
|
+
result.topics = result.topics.filter((topic) => topic.name !== '__consumer_offsets');
|
|
71
|
+
result.topics.forEach((topic) => {
|
|
72
|
+
topic.topicId = 'Any<UUID>';
|
|
73
|
+
topic.partitions.forEach((partition) => {
|
|
74
|
+
partition.leaderId = 0;
|
|
75
|
+
partition.isrNodes = [0];
|
|
76
|
+
partition.replicaNodes = [0];
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
80
|
+
});
|
|
81
|
+
let partitionIndex = 0;
|
|
82
|
+
let leaderId = 0;
|
|
83
|
+
(0, vitest_1.it)('should request metadata for a topic', async () => {
|
|
84
|
+
const result = await cluster.sendRequest(api_1.API.METADATA, {
|
|
85
|
+
topics: [{ id: topicId, name: 'kafka-ts-test-topic' }],
|
|
86
|
+
allowTopicAutoCreation: false,
|
|
87
|
+
includeTopicAuthorizedOperations: false,
|
|
88
|
+
});
|
|
89
|
+
partitionIndex = result.topics[0].partitions[0].partitionIndex;
|
|
90
|
+
leaderId = result.topics[0].partitions[0].leaderId;
|
|
91
|
+
result.controllerId = 0;
|
|
92
|
+
result.topics.forEach((topic) => {
|
|
93
|
+
topic.topicId = 'Any<UUID>';
|
|
94
|
+
topic.partitions.forEach((partition) => {
|
|
95
|
+
partition.leaderId = 0;
|
|
96
|
+
partition.isrNodes = [0];
|
|
97
|
+
partition.replicaNodes = [0];
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
101
|
+
});
|
|
102
|
+
let producerId = 9n;
|
|
103
|
+
(0, vitest_1.it)('should init producer id', async () => {
|
|
104
|
+
const result = await cluster.sendRequest(api_1.API.INIT_PRODUCER_ID, {
|
|
105
|
+
transactionalId: null,
|
|
106
|
+
transactionTimeoutMs: 0,
|
|
107
|
+
producerId,
|
|
108
|
+
producerEpoch: 0,
|
|
109
|
+
});
|
|
110
|
+
result.producerId = 0n;
|
|
111
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
112
|
+
});
|
|
113
|
+
(0, vitest_1.it)('should produce messages', async () => {
|
|
114
|
+
const now = Date.now();
|
|
115
|
+
const result = await cluster.sendRequestToNode(leaderId)(api_1.API.PRODUCE, {
|
|
116
|
+
transactionalId: null,
|
|
117
|
+
timeoutMs: 10000,
|
|
118
|
+
acks: 1,
|
|
119
|
+
topicData: [
|
|
120
|
+
{
|
|
121
|
+
name: 'kafka-ts-test-topic',
|
|
122
|
+
partitionData: [
|
|
123
|
+
{
|
|
124
|
+
index: partitionIndex,
|
|
125
|
+
baseOffset: 0n,
|
|
126
|
+
partitionLeaderEpoch: 0,
|
|
127
|
+
attributes: 0,
|
|
128
|
+
baseSequence: 0,
|
|
129
|
+
baseTimestamp: BigInt(now),
|
|
130
|
+
lastOffsetDelta: 0,
|
|
131
|
+
maxTimestamp: BigInt(now),
|
|
132
|
+
producerEpoch: 0,
|
|
133
|
+
producerId,
|
|
134
|
+
records: [
|
|
135
|
+
{
|
|
136
|
+
attributes: 0,
|
|
137
|
+
offsetDelta: 0,
|
|
138
|
+
timestampDelta: 0n,
|
|
139
|
+
key: Buffer.from('key'),
|
|
140
|
+
value: Buffer.from('value'),
|
|
141
|
+
headers: [
|
|
142
|
+
{
|
|
143
|
+
key: Buffer.from('header-key'),
|
|
144
|
+
value: Buffer.from('header-value'),
|
|
145
|
+
},
|
|
146
|
+
],
|
|
147
|
+
},
|
|
148
|
+
],
|
|
149
|
+
},
|
|
150
|
+
],
|
|
151
|
+
},
|
|
152
|
+
],
|
|
153
|
+
});
|
|
154
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
155
|
+
});
|
|
156
|
+
(0, vitest_1.it)('should fetch messages', async () => {
|
|
157
|
+
const result = await cluster.sendRequestToNode(leaderId)(api_1.API.FETCH, {
|
|
158
|
+
maxWaitMs: 100,
|
|
159
|
+
minBytes: 1,
|
|
160
|
+
maxBytes: 10485760,
|
|
161
|
+
isolationLevel: 1,
|
|
162
|
+
sessionId: 0,
|
|
163
|
+
sessionEpoch: -1,
|
|
164
|
+
topics: [
|
|
165
|
+
{
|
|
166
|
+
topicId,
|
|
167
|
+
partitions: [
|
|
168
|
+
{
|
|
169
|
+
partition: partitionIndex,
|
|
170
|
+
currentLeaderEpoch: -1,
|
|
171
|
+
fetchOffset: 0n,
|
|
172
|
+
lastFetchedEpoch: 0,
|
|
173
|
+
logStartOffset: -1n,
|
|
174
|
+
partitionMaxBytes: 10485760,
|
|
175
|
+
},
|
|
176
|
+
],
|
|
177
|
+
},
|
|
178
|
+
],
|
|
179
|
+
forgottenTopicsData: [],
|
|
180
|
+
rackId: '',
|
|
181
|
+
});
|
|
182
|
+
result.responses.forEach((response) => {
|
|
183
|
+
response.topicId = 'Any<UUID>';
|
|
184
|
+
response.partitions.forEach((partition) => {
|
|
185
|
+
partition.records.forEach((record) => {
|
|
186
|
+
(0, vitest_1.expect)(record.baseTimestamp).toBeGreaterThan(1721926744730n);
|
|
187
|
+
(0, vitest_1.expect)(record.maxTimestamp).toBeGreaterThan(1721926744730n);
|
|
188
|
+
(0, vitest_1.expect)(record.crc).toBeGreaterThan(0);
|
|
189
|
+
record.baseTimestamp = 0n;
|
|
190
|
+
record.maxTimestamp = 0n;
|
|
191
|
+
record.crc = 0;
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
196
|
+
});
|
|
197
|
+
let coordinatorId = -1;
|
|
198
|
+
(0, vitest_1.it)('should find coordinator', async () => {
|
|
199
|
+
const result = await cluster.sendRequest(api_1.API.FIND_COORDINATOR, { keyType: find_coordinator_1.KEY_TYPE.GROUP, keys: [groupId] });
|
|
200
|
+
result.coordinators.forEach((coordinator) => {
|
|
201
|
+
coordinator.key = 'Any<String>';
|
|
202
|
+
});
|
|
203
|
+
coordinatorId = result.coordinators[0].nodeId;
|
|
204
|
+
result.coordinators.forEach((coordinator) => {
|
|
205
|
+
coordinator.nodeId = 1;
|
|
206
|
+
coordinator.port = 9093;
|
|
207
|
+
});
|
|
208
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
209
|
+
});
|
|
210
|
+
let memberId = '';
|
|
211
|
+
(0, vitest_1.it)('should fail join group request with new memberId', async () => {
|
|
212
|
+
try {
|
|
213
|
+
const result = await cluster.sendRequestToNode(coordinatorId)(api_1.API.JOIN_GROUP, {
|
|
214
|
+
groupId,
|
|
215
|
+
sessionTimeoutMs: 30000,
|
|
216
|
+
rebalanceTimeoutMs: 60000,
|
|
217
|
+
memberId,
|
|
218
|
+
groupInstanceId: null,
|
|
219
|
+
protocolType: 'consumer',
|
|
220
|
+
protocols: [
|
|
221
|
+
{
|
|
222
|
+
name: 'RoundRobinAssigner',
|
|
223
|
+
metadata: { version: 0, topics: ['kafka-ts-test-topic'] },
|
|
224
|
+
},
|
|
225
|
+
],
|
|
226
|
+
reason: null,
|
|
227
|
+
});
|
|
228
|
+
(0, vitest_1.expect)(false, 'Should throw an error').toBe(true);
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
const { response } = error;
|
|
232
|
+
memberId = response.memberId;
|
|
233
|
+
response.memberId = 'Any<UUID>';
|
|
234
|
+
(0, vitest_1.expect)(response).toMatchSnapshot();
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
(0, vitest_1.it)('should join group', async () => {
|
|
238
|
+
const result = await cluster.sendRequestToNode(coordinatorId)(api_1.API.JOIN_GROUP, {
|
|
239
|
+
groupId,
|
|
240
|
+
sessionTimeoutMs: 30000,
|
|
241
|
+
rebalanceTimeoutMs: 60000,
|
|
242
|
+
memberId,
|
|
243
|
+
groupInstanceId: null,
|
|
244
|
+
protocolType: 'consumer',
|
|
245
|
+
protocols: [
|
|
246
|
+
{
|
|
247
|
+
name: 'RoundRobinAssigner',
|
|
248
|
+
metadata: { version: 0, topics: ['kafka-ts-test-topic'] },
|
|
249
|
+
},
|
|
250
|
+
],
|
|
251
|
+
reason: null,
|
|
252
|
+
});
|
|
253
|
+
result.memberId = 'Any<UUID>';
|
|
254
|
+
result.leader = 'Any<UUID>';
|
|
255
|
+
result.members.forEach((member) => {
|
|
256
|
+
member.memberId = 'Any<UUID>';
|
|
257
|
+
});
|
|
258
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
259
|
+
});
|
|
260
|
+
(0, vitest_1.it)('should sync group', async () => {
|
|
261
|
+
const result = await cluster.sendRequestToNode(coordinatorId)(api_1.API.SYNC_GROUP, {
|
|
262
|
+
groupId,
|
|
263
|
+
generationId: 1,
|
|
264
|
+
memberId,
|
|
265
|
+
groupInstanceId: null,
|
|
266
|
+
protocolType: 'consumer',
|
|
267
|
+
protocolName: 'RoundRobinAssigner',
|
|
268
|
+
assignments: [
|
|
269
|
+
{
|
|
270
|
+
memberId,
|
|
271
|
+
assignment: { 'kafka-test-topic': [0] },
|
|
272
|
+
},
|
|
273
|
+
],
|
|
274
|
+
});
|
|
275
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
276
|
+
});
|
|
277
|
+
(0, vitest_1.it)('should commit offsets', async () => {
|
|
278
|
+
const result = await cluster.sendRequestToNode(coordinatorId)(api_1.API.OFFSET_COMMIT, {
|
|
279
|
+
groupId,
|
|
280
|
+
generationIdOrMemberEpoch: 1,
|
|
281
|
+
memberId,
|
|
282
|
+
groupInstanceId: null,
|
|
283
|
+
topics: [
|
|
284
|
+
{
|
|
285
|
+
name: 'kafka-ts-test-topic',
|
|
286
|
+
partitions: [
|
|
287
|
+
{ partitionIndex: 0, committedOffset: 1n, committedLeaderEpoch: 0, committedMetadata: null },
|
|
288
|
+
],
|
|
289
|
+
},
|
|
290
|
+
],
|
|
291
|
+
});
|
|
292
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
293
|
+
});
|
|
294
|
+
(0, vitest_1.it)('should fetch offsets', async () => {
|
|
295
|
+
const result = await cluster.sendRequestToNode(coordinatorId)(api_1.API.OFFSET_FETCH, {
|
|
296
|
+
groups: [
|
|
297
|
+
{
|
|
298
|
+
groupId,
|
|
299
|
+
topics: [
|
|
300
|
+
{
|
|
301
|
+
name: 'kafka-ts-test-topic',
|
|
302
|
+
partitionIndexes: [0],
|
|
303
|
+
},
|
|
304
|
+
],
|
|
305
|
+
},
|
|
306
|
+
],
|
|
307
|
+
requireStable: false,
|
|
308
|
+
});
|
|
309
|
+
result.groups.forEach((group) => {
|
|
310
|
+
group.groupId = 'Any<String>';
|
|
311
|
+
});
|
|
312
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
313
|
+
});
|
|
314
|
+
(0, vitest_1.it)('should heartbeat', async () => {
|
|
315
|
+
const result = await cluster.sendRequestToNode(coordinatorId)(api_1.API.HEARTBEAT, {
|
|
316
|
+
groupId,
|
|
317
|
+
generationId: 1,
|
|
318
|
+
memberId,
|
|
319
|
+
groupInstanceId: null,
|
|
320
|
+
});
|
|
321
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
322
|
+
});
|
|
323
|
+
(0, vitest_1.it)('should leave group', async () => {
|
|
324
|
+
const result = await cluster.sendRequestToNode(coordinatorId)(api_1.API.LEAVE_GROUP, {
|
|
325
|
+
groupId,
|
|
326
|
+
members: [{ memberId, groupInstanceId: null, reason: null }],
|
|
327
|
+
});
|
|
328
|
+
result.members.forEach((member) => {
|
|
329
|
+
member.memberId = 'Any<UUID>';
|
|
330
|
+
});
|
|
331
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
332
|
+
});
|
|
333
|
+
(0, vitest_1.it)('should delete topics', async () => {
|
|
334
|
+
const result = await cluster.sendRequest(api_1.API.DELETE_TOPICS, {
|
|
335
|
+
topics: [{ name: 'kafka-ts-test-topic', topicId: null }],
|
|
336
|
+
timeoutMs: 10000,
|
|
337
|
+
});
|
|
338
|
+
result.responses.forEach((response) => {
|
|
339
|
+
response.topicId = 'Any<UUID>';
|
|
340
|
+
});
|
|
341
|
+
(0, vitest_1.expect)(result).toMatchSnapshot();
|
|
342
|
+
});
|
|
343
|
+
});
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.GZIP = void 0;
|
|
4
|
+
const zlib_1 = require("zlib");
|
|
5
|
+
exports.GZIP = {
|
|
6
|
+
compress: async (data) => new Promise((resolve, reject) => (0, zlib_1.gzip)(data, (err, result) => (err ? reject(err) : resolve(result)))),
|
|
7
|
+
decompress: async (data) => new Promise((resolve, reject) => (0, zlib_1.unzip)(data, (err, result) => (err ? reject(err) : resolve(result)))),
|
|
8
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.findCodec = void 0;
|
|
4
|
+
const gzip_1 = require("./gzip");
|
|
5
|
+
const none_1 = require("./none");
|
|
6
|
+
const codecs = {
|
|
7
|
+
0: none_1.NONE,
|
|
8
|
+
1: gzip_1.GZIP,
|
|
9
|
+
};
|
|
10
|
+
const findCodec = (type) => {
|
|
11
|
+
const codec = codecs[type];
|
|
12
|
+
if (!codec) {
|
|
13
|
+
throw new Error(`Unsupported codec: ${type}`);
|
|
14
|
+
}
|
|
15
|
+
return codec;
|
|
16
|
+
};
|
|
17
|
+
exports.findCodec = findCodec;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
import { TcpSocketConnectOpts } from 'net';
|
|
4
|
+
import { TLSSocketOptions } from 'tls';
|
|
5
|
+
import { Api } from './utils/api';
|
|
6
|
+
type ConnectionOptions = {
|
|
7
|
+
clientId: string | null;
|
|
8
|
+
connection: TcpSocketConnectOpts;
|
|
9
|
+
ssl: TLSSocketOptions | null;
|
|
10
|
+
};
|
|
11
|
+
export declare class Connection {
|
|
12
|
+
private options;
|
|
13
|
+
private socket;
|
|
14
|
+
private queue;
|
|
15
|
+
private lastCorrelationId;
|
|
16
|
+
private chunks;
|
|
17
|
+
constructor(options: ConnectionOptions);
|
|
18
|
+
connect(): Promise<void>;
|
|
19
|
+
disconnect(): Promise<void>;
|
|
20
|
+
sendRequest<Request, Response>(api: Api<Request, Response>, body: Request): Promise<Response>;
|
|
21
|
+
private write;
|
|
22
|
+
private handleData;
|
|
23
|
+
private nextCorrelationId;
|
|
24
|
+
}
|
|
25
|
+
export type SendRequest = typeof Connection.prototype.sendRequest;
|
|
26
|
+
export {};
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
19
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
20
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
21
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
22
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
23
|
+
};
|
|
24
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
25
|
+
if (mod && mod.__esModule) return mod;
|
|
26
|
+
var result = {};
|
|
27
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
28
|
+
__setModuleDefault(result, mod);
|
|
29
|
+
return result;
|
|
30
|
+
};
|
|
31
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
32
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
33
|
+
};
|
|
34
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
35
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
36
|
+
};
|
|
37
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
38
|
+
exports.Connection = void 0;
|
|
39
|
+
const assert_1 = __importDefault(require("assert"));
|
|
40
|
+
const net_1 = __importStar(require("net"));
|
|
41
|
+
const tls_1 = __importDefault(require("tls"));
|
|
42
|
+
const api_1 = require("./api");
|
|
43
|
+
const decoder_1 = require("./utils/decoder");
|
|
44
|
+
const encoder_1 = require("./utils/encoder");
|
|
45
|
+
const error_1 = require("./utils/error");
|
|
46
|
+
const logger_1 = require("./utils/logger");
|
|
47
|
+
const tracer_1 = require("./utils/tracer");
|
|
48
|
+
const trace = (0, tracer_1.createTracer)('Connection');
|
|
49
|
+
class Connection {
|
|
50
|
+
options;
|
|
51
|
+
socket = new net_1.Socket();
|
|
52
|
+
queue = {};
|
|
53
|
+
lastCorrelationId = 0;
|
|
54
|
+
chunks = [];
|
|
55
|
+
constructor(options) {
|
|
56
|
+
this.options = options;
|
|
57
|
+
}
|
|
58
|
+
async connect() {
|
|
59
|
+
this.queue = {};
|
|
60
|
+
this.chunks = [];
|
|
61
|
+
await new Promise((resolve, reject) => {
|
|
62
|
+
const { ssl, connection } = this.options;
|
|
63
|
+
this.socket = ssl
|
|
64
|
+
? tls_1.default.connect({
|
|
65
|
+
...connection,
|
|
66
|
+
...ssl,
|
|
67
|
+
...(connection.host && !(0, net_1.isIP)(connection.host) && { servername: connection.host }),
|
|
68
|
+
}, resolve)
|
|
69
|
+
: net_1.default.connect(connection, resolve);
|
|
70
|
+
this.socket.once('error', reject);
|
|
71
|
+
});
|
|
72
|
+
this.socket.removeAllListeners('error');
|
|
73
|
+
this.socket.on('data', (data) => this.handleData(data));
|
|
74
|
+
this.socket.once('close', async () => {
|
|
75
|
+
Object.values(this.queue).forEach(({ reject }) => {
|
|
76
|
+
reject(new error_1.ConnectionError('Socket closed unexpectedly'));
|
|
77
|
+
});
|
|
78
|
+
this.queue = {};
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
disconnect() {
|
|
82
|
+
this.socket.removeAllListeners();
|
|
83
|
+
return new Promise((resolve) => {
|
|
84
|
+
if (this.socket.pending) {
|
|
85
|
+
return resolve();
|
|
86
|
+
}
|
|
87
|
+
this.socket.end(resolve);
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
async sendRequest(api, body) {
|
|
91
|
+
const correlationId = this.nextCorrelationId();
|
|
92
|
+
const apiName = (0, api_1.getApiName)(api);
|
|
93
|
+
const encoder = new encoder_1.Encoder()
|
|
94
|
+
.writeInt16(api.apiKey)
|
|
95
|
+
.writeInt16(api.apiVersion)
|
|
96
|
+
.writeInt32(correlationId)
|
|
97
|
+
.writeString(this.options.clientId);
|
|
98
|
+
const request = api.request(encoder, body);
|
|
99
|
+
const requestEncoder = new encoder_1.Encoder().writeInt32(request.getByteLength()).writeEncoder(request);
|
|
100
|
+
let timeout;
|
|
101
|
+
const { responseDecoder, responseSize } = await new Promise(async (resolve, reject) => {
|
|
102
|
+
timeout = setTimeout(() => {
|
|
103
|
+
delete this.queue[correlationId];
|
|
104
|
+
reject(new error_1.ConnectionError(`${apiName} timed out`));
|
|
105
|
+
}, 30_000);
|
|
106
|
+
try {
|
|
107
|
+
this.queue[correlationId] = { resolve, reject };
|
|
108
|
+
await this.write(requestEncoder.value());
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
reject(error);
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
clearTimeout(timeout);
|
|
115
|
+
const response = await api.response(responseDecoder);
|
|
116
|
+
(0, assert_1.default)(responseDecoder.getOffset() - 4 === responseSize, `Buffer not correctly consumed: ${responseDecoder.getOffset() - 4} !== ${responseSize}`);
|
|
117
|
+
return response;
|
|
118
|
+
}
|
|
119
|
+
write(buffer) {
|
|
120
|
+
return new Promise((resolve, reject) => {
|
|
121
|
+
const { stack } = new Error('Write error');
|
|
122
|
+
this.socket.write(buffer, (error) => {
|
|
123
|
+
if (error) {
|
|
124
|
+
const err = new error_1.ConnectionError(error.message);
|
|
125
|
+
err.stack += `\n${stack}`;
|
|
126
|
+
return reject(err);
|
|
127
|
+
}
|
|
128
|
+
resolve();
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
handleData(buffer) {
|
|
133
|
+
this.chunks.push(buffer);
|
|
134
|
+
const decoder = new decoder_1.Decoder(Buffer.concat(this.chunks));
|
|
135
|
+
if (decoder.getBufferLength() < 4) {
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
const size = decoder.readInt32();
|
|
139
|
+
if (size !== decoder.getBufferLength() - 4) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
const correlationId = decoder.readInt32();
|
|
143
|
+
const context = this.queue[correlationId];
|
|
144
|
+
if (context) {
|
|
145
|
+
delete this.queue[correlationId];
|
|
146
|
+
context.resolve({ responseDecoder: decoder, responseSize: size });
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
logger_1.log.debug('Could not find pending request for correlationId', { correlationId });
|
|
150
|
+
}
|
|
151
|
+
this.chunks = [];
|
|
152
|
+
}
|
|
153
|
+
nextCorrelationId() {
|
|
154
|
+
return this.lastCorrelationId++;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
exports.Connection = Connection;
|
|
158
|
+
__decorate([
|
|
159
|
+
trace(),
|
|
160
|
+
__metadata("design:type", Function),
|
|
161
|
+
__metadata("design:paramtypes", []),
|
|
162
|
+
__metadata("design:returntype", Promise)
|
|
163
|
+
], Connection.prototype, "connect", null);
|
|
164
|
+
__decorate([
|
|
165
|
+
trace(),
|
|
166
|
+
__metadata("design:type", Function),
|
|
167
|
+
__metadata("design:paramtypes", []),
|
|
168
|
+
__metadata("design:returntype", void 0)
|
|
169
|
+
], Connection.prototype, "disconnect", null);
|
|
170
|
+
__decorate([
|
|
171
|
+
trace((api, body) => ({ message: (0, api_1.getApiName)(api), body })),
|
|
172
|
+
__metadata("design:type", Function),
|
|
173
|
+
__metadata("design:paramtypes", [Object, Request]),
|
|
174
|
+
__metadata("design:returntype", Promise)
|
|
175
|
+
], Connection.prototype, "sendRequest", null);
|