kafka-ts 0.0.1-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierrc +7 -0
- package/LICENSE +24 -0
- package/README.md +88 -0
- package/certs/ca.crt +29 -0
- package/certs/ca.key +52 -0
- package/certs/ca.srl +1 -0
- package/certs/kafka.crt +29 -0
- package/certs/kafka.csr +26 -0
- package/certs/kafka.key +52 -0
- package/certs/kafka.keystore.jks +0 -0
- package/certs/kafka.truststore.jks +0 -0
- package/dist/api/api-versions.d.ts +9 -0
- package/dist/api/api-versions.js +24 -0
- package/dist/api/create-topics.d.ts +38 -0
- package/dist/api/create-topics.js +53 -0
- package/dist/api/delete-topics.d.ts +18 -0
- package/dist/api/delete-topics.js +33 -0
- package/dist/api/fetch.d.ts +77 -0
- package/dist/api/fetch.js +106 -0
- package/dist/api/find-coordinator.d.ts +21 -0
- package/dist/api/find-coordinator.js +39 -0
- package/dist/api/heartbeat.d.ts +11 -0
- package/dist/api/heartbeat.js +27 -0
- package/dist/api/index.d.ts +573 -0
- package/dist/api/index.js +164 -0
- package/dist/api/init-producer-id.d.ts +13 -0
- package/dist/api/init-producer-id.js +29 -0
- package/dist/api/join-group.d.ts +34 -0
- package/dist/api/join-group.js +51 -0
- package/dist/api/leave-group.d.ts +19 -0
- package/dist/api/leave-group.js +39 -0
- package/dist/api/list-offsets.d.ts +29 -0
- package/dist/api/list-offsets.js +48 -0
- package/dist/api/metadata.d.ts +40 -0
- package/dist/api/metadata.js +58 -0
- package/dist/api/offset-commit.d.ts +28 -0
- package/dist/api/offset-commit.js +48 -0
- package/dist/api/offset-fetch.d.ts +33 -0
- package/dist/api/offset-fetch.js +57 -0
- package/dist/api/produce.d.ts +53 -0
- package/dist/api/produce.js +129 -0
- package/dist/api/sasl-authenticate.d.ts +11 -0
- package/dist/api/sasl-authenticate.js +23 -0
- package/dist/api/sasl-handshake.d.ts +6 -0
- package/dist/api/sasl-handshake.js +19 -0
- package/dist/api/sync-group.d.ts +24 -0
- package/dist/api/sync-group.js +36 -0
- package/dist/broker.d.ts +29 -0
- package/dist/broker.js +60 -0
- package/dist/client.d.ts +23 -0
- package/dist/client.js +36 -0
- package/dist/cluster.d.ts +24 -0
- package/dist/cluster.js +72 -0
- package/dist/connection.d.ts +25 -0
- package/dist/connection.js +155 -0
- package/dist/consumer/consumer-group.d.ts +36 -0
- package/dist/consumer/consumer-group.js +182 -0
- package/dist/consumer/consumer-metadata.d.ts +7 -0
- package/dist/consumer/consumer-metadata.js +14 -0
- package/dist/consumer/consumer.d.ts +37 -0
- package/dist/consumer/consumer.js +178 -0
- package/dist/consumer/metadata.d.ts +24 -0
- package/dist/consumer/metadata.js +64 -0
- package/dist/consumer/offset-manager.d.ts +22 -0
- package/dist/consumer/offset-manager.js +56 -0
- package/dist/distributors/assignments-to-replicas.d.ts +17 -0
- package/dist/distributors/assignments-to-replicas.js +60 -0
- package/dist/distributors/assignments-to-replicas.test.d.ts +1 -0
- package/dist/distributors/assignments-to-replicas.test.js +40 -0
- package/dist/distributors/messages-to-topic-partition-leaders.d.ts +17 -0
- package/dist/distributors/messages-to-topic-partition-leaders.js +15 -0
- package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts +1 -0
- package/dist/distributors/messages-to-topic-partition-leaders.test.js +30 -0
- package/dist/examples/src/replicator.js +34 -0
- package/dist/examples/src/utils/json.js +5 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +19 -0
- package/dist/metadata.d.ts +24 -0
- package/dist/metadata.js +89 -0
- package/dist/producer/producer.d.ts +19 -0
- package/dist/producer/producer.js +111 -0
- package/dist/request-handler.d.ts +16 -0
- package/dist/request-handler.js +67 -0
- package/dist/request-handler.test.d.ts +1 -0
- package/dist/request-handler.test.js +340 -0
- package/dist/src/api/api-versions.js +18 -0
- package/dist/src/api/create-topics.js +46 -0
- package/dist/src/api/delete-topics.js +26 -0
- package/dist/src/api/fetch.js +95 -0
- package/dist/src/api/find-coordinator.js +34 -0
- package/dist/src/api/heartbeat.js +22 -0
- package/dist/src/api/index.js +38 -0
- package/dist/src/api/init-producer-id.js +24 -0
- package/dist/src/api/join-group.js +48 -0
- package/dist/src/api/leave-group.js +30 -0
- package/dist/src/api/list-offsets.js +39 -0
- package/dist/src/api/metadata.js +47 -0
- package/dist/src/api/offset-commit.js +39 -0
- package/dist/src/api/offset-fetch.js +44 -0
- package/dist/src/api/produce.js +119 -0
- package/dist/src/api/sync-group.js +31 -0
- package/dist/src/broker.js +35 -0
- package/dist/src/connection.js +21 -0
- package/dist/src/consumer/consumer-group.js +131 -0
- package/dist/src/consumer/consumer.js +103 -0
- package/dist/src/consumer/metadata.js +52 -0
- package/dist/src/consumer/offset-manager.js +23 -0
- package/dist/src/index.js +19 -0
- package/dist/src/producer/producer.js +84 -0
- package/dist/src/request-handler.js +57 -0
- package/dist/src/request-handler.test.js +321 -0
- package/dist/src/types.js +2 -0
- package/dist/src/utils/api.js +5 -0
- package/dist/src/utils/decoder.js +161 -0
- package/dist/src/utils/encoder.js +137 -0
- package/dist/src/utils/error.js +10 -0
- package/dist/types.d.ts +9 -0
- package/dist/types.js +2 -0
- package/dist/utils/api.d.ts +9 -0
- package/dist/utils/api.js +5 -0
- package/dist/utils/debug.d.ts +2 -0
- package/dist/utils/debug.js +11 -0
- package/dist/utils/decoder.d.ts +29 -0
- package/dist/utils/decoder.js +147 -0
- package/dist/utils/delay.d.ts +1 -0
- package/dist/utils/delay.js +5 -0
- package/dist/utils/encoder.d.ts +28 -0
- package/dist/utils/encoder.js +122 -0
- package/dist/utils/error.d.ts +11 -0
- package/dist/utils/error.js +27 -0
- package/dist/utils/memo.d.ts +1 -0
- package/dist/utils/memo.js +16 -0
- package/dist/utils/retrier.d.ts +10 -0
- package/dist/utils/retrier.js +22 -0
- package/dist/utils/tracer.d.ts +1 -0
- package/dist/utils/tracer.js +26 -0
- package/docker-compose.yml +104 -0
- package/examples/node_modules/.package-lock.json +22 -0
- package/examples/package-lock.json +30 -0
- package/examples/package.json +14 -0
- package/examples/src/client.ts +9 -0
- package/examples/src/consumer.ts +17 -0
- package/examples/src/create-topic.ts +37 -0
- package/examples/src/producer.ts +24 -0
- package/examples/src/replicator.ts +25 -0
- package/examples/src/utils/json.ts +1 -0
- package/examples/tsconfig.json +7 -0
- package/log4j.properties +95 -0
- package/package.json +17 -0
- package/scripts/generate-certs.sh +24 -0
- package/src/__snapshots__/request-handler.test.ts.snap +1687 -0
- package/src/api/api-versions.ts +21 -0
- package/src/api/create-topics.ts +78 -0
- package/src/api/delete-topics.ts +42 -0
- package/src/api/fetch.ts +143 -0
- package/src/api/find-coordinator.ts +39 -0
- package/src/api/heartbeat.ts +33 -0
- package/src/api/index.ts +164 -0
- package/src/api/init-producer-id.ts +35 -0
- package/src/api/join-group.ts +67 -0
- package/src/api/leave-group.ts +48 -0
- package/src/api/list-offsets.ts +65 -0
- package/src/api/metadata.ts +66 -0
- package/src/api/offset-commit.ts +67 -0
- package/src/api/offset-fetch.ts +74 -0
- package/src/api/produce.ts +173 -0
- package/src/api/sasl-authenticate.ts +21 -0
- package/src/api/sasl-handshake.ts +16 -0
- package/src/api/sync-group.ts +54 -0
- package/src/broker.ts +74 -0
- package/src/client.ts +47 -0
- package/src/cluster.ts +87 -0
- package/src/connection.ts +141 -0
- package/src/consumer/consumer-group.ts +209 -0
- package/src/consumer/consumer-metadata.ts +14 -0
- package/src/consumer/consumer.ts +229 -0
- package/src/consumer/offset-manager.ts +93 -0
- package/src/distributors/assignments-to-replicas.test.ts +43 -0
- package/src/distributors/assignments-to-replicas.ts +85 -0
- package/src/distributors/messages-to-topic-partition-leaders.test.ts +32 -0
- package/src/distributors/messages-to-topic-partition-leaders.ts +19 -0
- package/src/index.ts +3 -0
- package/src/metadata.ts +122 -0
- package/src/producer/producer.ts +132 -0
- package/src/request-handler.test.ts +366 -0
- package/src/types.ts +9 -0
- package/src/utils/api.ts +11 -0
- package/src/utils/debug.ts +9 -0
- package/src/utils/decoder.ts +168 -0
- package/src/utils/delay.ts +1 -0
- package/src/utils/encoder.ts +141 -0
- package/src/utils/error.ts +21 -0
- package/src/utils/memo.ts +12 -0
- package/src/utils/retrier.ts +39 -0
- package/src/utils/tracer.ts +28 -0
- package/tsconfig.json +17 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FETCH = void 0;
|
|
4
|
+
const api_1 = require("../utils/api");
|
|
5
|
+
const error_1 = require("../utils/error");
|
|
6
|
+
exports.FETCH = (0, api_1.createApi)({
|
|
7
|
+
apiKey: 1,
|
|
8
|
+
apiVersion: 16,
|
|
9
|
+
request: (encoder, data) => encoder
|
|
10
|
+
.writeUVarInt(0)
|
|
11
|
+
.writeInt32(data.maxWaitMs)
|
|
12
|
+
.writeInt32(data.minBytes)
|
|
13
|
+
.writeInt32(data.maxBytes)
|
|
14
|
+
.writeInt8(data.isolationLevel)
|
|
15
|
+
.writeInt32(data.sessionId)
|
|
16
|
+
.writeInt32(data.sessionEpoch)
|
|
17
|
+
.writeCompactArray(data.topics, (encoder, topic) => encoder
|
|
18
|
+
.writeUUID(topic.topicId)
|
|
19
|
+
.writeCompactArray(topic.partitions, (encoder, partition) => encoder
|
|
20
|
+
.writeInt32(partition.partition)
|
|
21
|
+
.writeInt32(partition.currentLeaderEpoch)
|
|
22
|
+
.writeInt64(partition.fetchOffset)
|
|
23
|
+
.writeInt32(partition.lastFetchedEpoch)
|
|
24
|
+
.writeInt64(partition.logStartOffset)
|
|
25
|
+
.writeInt32(partition.partitionMaxBytes)
|
|
26
|
+
.writeUVarInt(0))
|
|
27
|
+
.writeUVarInt(0))
|
|
28
|
+
.writeCompactArray(data.forgottenTopicsData, (encoder, forgottenTopic) => encoder
|
|
29
|
+
.writeUUID(forgottenTopic.topicId)
|
|
30
|
+
.writeCompactArray(forgottenTopic.partitions, (encoder, partition) => encoder.writeInt32(partition))
|
|
31
|
+
.writeUVarInt(0))
|
|
32
|
+
.writeCompactString(data.rackId)
|
|
33
|
+
.writeUVarInt(0),
|
|
34
|
+
response: (decoder) => {
|
|
35
|
+
const result = {
|
|
36
|
+
_tag: decoder.readTagBuffer(),
|
|
37
|
+
throttleTimeMs: decoder.readInt32(),
|
|
38
|
+
errorCode: decoder.readInt16(),
|
|
39
|
+
sessionId: decoder.readInt32(),
|
|
40
|
+
responses: decoder.readCompactArray((response) => ({
|
|
41
|
+
topicId: response.readUUID(),
|
|
42
|
+
partitions: response.readCompactArray((partition) => ({
|
|
43
|
+
partitionIndex: partition.readInt32(),
|
|
44
|
+
errorCode: partition.readInt16(),
|
|
45
|
+
highWatermark: partition.readInt64(),
|
|
46
|
+
lastStableOffset: partition.readInt64(),
|
|
47
|
+
logStartOffset: partition.readInt64(),
|
|
48
|
+
abortedTransactions: partition.readCompactArray((abortedTransaction) => ({
|
|
49
|
+
producerId: abortedTransaction.readInt64(),
|
|
50
|
+
firstOffset: abortedTransaction.readInt64(),
|
|
51
|
+
_tag: abortedTransaction.readTagBuffer(),
|
|
52
|
+
})),
|
|
53
|
+
preferredReadReplica: partition.readInt32(),
|
|
54
|
+
records: decodeRecords(partition),
|
|
55
|
+
_tag: partition.readTagBuffer(),
|
|
56
|
+
})),
|
|
57
|
+
_tag: response.readTagBuffer(),
|
|
58
|
+
})),
|
|
59
|
+
_tag2: decoder.readTagBuffer(),
|
|
60
|
+
};
|
|
61
|
+
if (result.errorCode)
|
|
62
|
+
throw new error_1.KafkaTSApiError(result.errorCode, null, result);
|
|
63
|
+
result.responses.forEach((response) => {
|
|
64
|
+
response.partitions.forEach((partition) => {
|
|
65
|
+
if (partition.errorCode)
|
|
66
|
+
throw new error_1.KafkaTSApiError(partition.errorCode, null, result);
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
return result;
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
const decodeRecords = (decoder) => {
|
|
73
|
+
const size = decoder.readUVarInt() - 1;
|
|
74
|
+
if (size <= 0) {
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
const results = [];
|
|
78
|
+
while (decoder.getBufferLength() > decoder.getOffset() + 49) {
|
|
79
|
+
results.push({
|
|
80
|
+
baseOffset: decoder.readInt64(),
|
|
81
|
+
batchLength: decoder.readInt32(),
|
|
82
|
+
partitionLeaderEpoch: decoder.readInt32(),
|
|
83
|
+
magic: decoder.readInt8(),
|
|
84
|
+
crc: decoder.readUInt32(),
|
|
85
|
+
attributes: decoder.readInt16(),
|
|
86
|
+
lastOffsetDelta: decoder.readInt32(),
|
|
87
|
+
baseTimestamp: decoder.readInt64(),
|
|
88
|
+
maxTimestamp: decoder.readInt64(),
|
|
89
|
+
producerId: decoder.readInt64(),
|
|
90
|
+
producerEpoch: decoder.readInt16(),
|
|
91
|
+
baseSequence: decoder.readInt32(),
|
|
92
|
+
records: decoder.readRecords((record) => ({
|
|
93
|
+
attributes: record.readInt8(),
|
|
94
|
+
timestampDelta: record.readVarLong(),
|
|
95
|
+
offsetDelta: record.readVarInt(),
|
|
96
|
+
key: record.readVarIntString(),
|
|
97
|
+
value: record.readVarIntString(),
|
|
98
|
+
headers: record.readCompactArray((header) => ({
|
|
99
|
+
key: header.readVarIntString(),
|
|
100
|
+
value: header.readVarIntString(),
|
|
101
|
+
})),
|
|
102
|
+
})),
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
return results;
|
|
106
|
+
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export declare const KEY_TYPE: {
|
|
2
|
+
GROUP: number;
|
|
3
|
+
TRANSACTION: number;
|
|
4
|
+
};
|
|
5
|
+
export declare const FIND_COORDINATOR: import("../utils/api").Api<{
|
|
6
|
+
keyType: number;
|
|
7
|
+
keys: string[];
|
|
8
|
+
}, {
|
|
9
|
+
_tag: void;
|
|
10
|
+
throttleTimeMs: number;
|
|
11
|
+
coordinators: {
|
|
12
|
+
key: string | null;
|
|
13
|
+
nodeId: number;
|
|
14
|
+
host: string;
|
|
15
|
+
port: number;
|
|
16
|
+
errorCode: number;
|
|
17
|
+
errorMessage: string | null;
|
|
18
|
+
_tag: void;
|
|
19
|
+
}[];
|
|
20
|
+
_tag2: void;
|
|
21
|
+
}>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FIND_COORDINATOR = exports.KEY_TYPE = void 0;
|
|
4
|
+
const api_1 = require("../utils/api");
|
|
5
|
+
const error_1 = require("../utils/error");
|
|
6
|
+
exports.KEY_TYPE = {
|
|
7
|
+
GROUP: 0,
|
|
8
|
+
TRANSACTION: 1,
|
|
9
|
+
};
|
|
10
|
+
exports.FIND_COORDINATOR = (0, api_1.createApi)({
|
|
11
|
+
apiKey: 10,
|
|
12
|
+
apiVersion: 4,
|
|
13
|
+
request: (encoder, data) => encoder
|
|
14
|
+
.writeUVarInt(0)
|
|
15
|
+
.writeInt8(data.keyType)
|
|
16
|
+
.writeCompactArray(data.keys, (encoder, key) => encoder.writeCompactString(key))
|
|
17
|
+
.writeUVarInt(0),
|
|
18
|
+
response: (decoder) => {
|
|
19
|
+
const result = {
|
|
20
|
+
_tag: decoder.readTagBuffer(),
|
|
21
|
+
throttleTimeMs: decoder.readInt32(),
|
|
22
|
+
coordinators: decoder.readCompactArray((decoder) => ({
|
|
23
|
+
key: decoder.readCompactString(),
|
|
24
|
+
nodeId: decoder.readInt32(),
|
|
25
|
+
host: decoder.readCompactString(),
|
|
26
|
+
port: decoder.readInt32(),
|
|
27
|
+
errorCode: decoder.readInt16(),
|
|
28
|
+
errorMessage: decoder.readCompactString(),
|
|
29
|
+
_tag: decoder.readTagBuffer(),
|
|
30
|
+
})),
|
|
31
|
+
_tag2: decoder.readTagBuffer(),
|
|
32
|
+
};
|
|
33
|
+
result.coordinators.forEach((coordinator) => {
|
|
34
|
+
if (coordinator.errorCode)
|
|
35
|
+
throw new error_1.KafkaTSApiError(coordinator.errorCode, coordinator.errorMessage, result);
|
|
36
|
+
});
|
|
37
|
+
return result;
|
|
38
|
+
},
|
|
39
|
+
});
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HEARTBEAT = void 0;
|
|
4
|
+
const api_1 = require("../utils/api");
|
|
5
|
+
const error_1 = require("../utils/error");
|
|
6
|
+
exports.HEARTBEAT = (0, api_1.createApi)({
|
|
7
|
+
apiKey: 12,
|
|
8
|
+
apiVersion: 4,
|
|
9
|
+
request: (encoder, data) => encoder
|
|
10
|
+
.writeUVarInt(0)
|
|
11
|
+
.writeCompactString(data.groupId)
|
|
12
|
+
.writeInt32(data.generationId)
|
|
13
|
+
.writeCompactString(data.memberId)
|
|
14
|
+
.writeCompactString(data.groupInstanceId)
|
|
15
|
+
.writeUVarInt(0),
|
|
16
|
+
response: (decoder) => {
|
|
17
|
+
const result = {
|
|
18
|
+
_tag: decoder.readTagBuffer(),
|
|
19
|
+
throttleTimeMs: decoder.readInt32(),
|
|
20
|
+
errorCode: decoder.readInt16(),
|
|
21
|
+
_tag2: decoder.readTagBuffer(),
|
|
22
|
+
};
|
|
23
|
+
if (result.errorCode)
|
|
24
|
+
throw new error_1.KafkaTSApiError(result.errorCode, null, result);
|
|
25
|
+
return result;
|
|
26
|
+
},
|
|
27
|
+
});
|