kafka-ts 0.0.1-beta → 0.0.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/release.yml +17 -0
- package/.prettierrc +3 -2
- package/LICENSE +1 -1
- package/README.md +48 -35
- package/docker-compose.yml +108 -102
- package/examples/package-lock.json +28 -27
- package/examples/package.json +12 -12
- package/examples/src/client.ts +6 -6
- package/examples/src/consumer.ts +9 -8
- package/examples/src/create-topic.ts +22 -16
- package/examples/src/producer.ts +7 -7
- package/examples/src/replicator.ts +4 -4
- package/examples/src/utils/delay.ts +1 -0
- package/examples/src/utils/json.ts +1 -1
- package/examples/tsconfig.json +2 -2
- package/package.json +22 -15
- package/scripts/create-scram-user.sh +5 -0
- package/scripts/generate-certs.sh +1 -0
- package/scripts/kafka-local.properties +33 -0
- package/src/__snapshots__/request-handler.test.ts.snap +9 -718
- package/src/api/api-versions.ts +2 -2
- package/src/api/create-topics.ts +2 -2
- package/src/api/delete-topics.ts +2 -2
- package/src/api/fetch.ts +3 -3
- package/src/api/find-coordinator.ts +2 -2
- package/src/api/heartbeat.ts +2 -2
- package/src/api/index.ts +18 -18
- package/src/api/init-producer-id.ts +2 -2
- package/src/api/join-group.ts +3 -3
- package/src/api/leave-group.ts +2 -2
- package/src/api/list-offsets.ts +3 -3
- package/src/api/metadata.ts +3 -3
- package/src/api/offset-commit.ts +2 -2
- package/src/api/offset-fetch.ts +2 -2
- package/src/api/produce.ts +3 -3
- package/src/api/sasl-authenticate.ts +2 -2
- package/src/api/sasl-handshake.ts +2 -2
- package/src/api/sync-group.ts +2 -2
- package/src/broker.ts +83 -11
- package/src/client.ts +6 -6
- package/src/{request-handler.test.ts → cluster.test.ts} +72 -69
- package/src/cluster.ts +7 -7
- package/src/connection.ts +17 -15
- package/src/consumer/consumer-group.ts +14 -14
- package/src/consumer/consumer-metadata.ts +2 -2
- package/src/consumer/consumer.ts +84 -82
- package/src/consumer/fetch-manager.ts +179 -0
- package/src/consumer/fetcher.ts +57 -0
- package/src/consumer/offset-manager.ts +6 -6
- package/src/consumer/processor.ts +47 -0
- package/src/distributors/assignments-to-replicas.test.ts +7 -7
- package/src/distributors/assignments-to-replicas.ts +1 -1
- package/src/distributors/messages-to-topic-partition-leaders.test.ts +6 -6
- package/src/index.ts +4 -3
- package/src/metadata.ts +4 -4
- package/src/producer/producer.ts +8 -8
- package/src/types.ts +2 -0
- package/src/utils/api.ts +4 -4
- package/src/utils/crypto.ts +14 -0
- package/src/utils/debug.ts +2 -2
- package/src/utils/decoder.ts +4 -4
- package/src/utils/encoder.ts +6 -6
- package/src/utils/error.ts +3 -3
- package/src/utils/retrier.ts +1 -1
- package/src/utils/tracer.ts +7 -4
- package/tsconfig.json +16 -16
- package/dist/api/api-versions.d.ts +0 -9
- package/dist/api/api-versions.js +0 -24
- package/dist/api/create-topics.d.ts +0 -38
- package/dist/api/create-topics.js +0 -53
- package/dist/api/delete-topics.d.ts +0 -18
- package/dist/api/delete-topics.js +0 -33
- package/dist/api/fetch.d.ts +0 -77
- package/dist/api/fetch.js +0 -106
- package/dist/api/find-coordinator.d.ts +0 -21
- package/dist/api/find-coordinator.js +0 -39
- package/dist/api/heartbeat.d.ts +0 -11
- package/dist/api/heartbeat.js +0 -27
- package/dist/api/index.d.ts +0 -573
- package/dist/api/index.js +0 -164
- package/dist/api/init-producer-id.d.ts +0 -13
- package/dist/api/init-producer-id.js +0 -29
- package/dist/api/join-group.d.ts +0 -34
- package/dist/api/join-group.js +0 -51
- package/dist/api/leave-group.d.ts +0 -19
- package/dist/api/leave-group.js +0 -39
- package/dist/api/list-offsets.d.ts +0 -29
- package/dist/api/list-offsets.js +0 -48
- package/dist/api/metadata.d.ts +0 -40
- package/dist/api/metadata.js +0 -58
- package/dist/api/offset-commit.d.ts +0 -28
- package/dist/api/offset-commit.js +0 -48
- package/dist/api/offset-fetch.d.ts +0 -33
- package/dist/api/offset-fetch.js +0 -57
- package/dist/api/produce.d.ts +0 -53
- package/dist/api/produce.js +0 -129
- package/dist/api/sasl-authenticate.d.ts +0 -11
- package/dist/api/sasl-authenticate.js +0 -23
- package/dist/api/sasl-handshake.d.ts +0 -6
- package/dist/api/sasl-handshake.js +0 -19
- package/dist/api/sync-group.d.ts +0 -24
- package/dist/api/sync-group.js +0 -36
- package/dist/broker.d.ts +0 -29
- package/dist/broker.js +0 -60
- package/dist/client.d.ts +0 -23
- package/dist/client.js +0 -36
- package/dist/cluster.d.ts +0 -24
- package/dist/cluster.js +0 -72
- package/dist/connection.d.ts +0 -25
- package/dist/connection.js +0 -155
- package/dist/consumer/consumer-group.d.ts +0 -36
- package/dist/consumer/consumer-group.js +0 -182
- package/dist/consumer/consumer-metadata.d.ts +0 -7
- package/dist/consumer/consumer-metadata.js +0 -14
- package/dist/consumer/consumer.d.ts +0 -37
- package/dist/consumer/consumer.js +0 -178
- package/dist/consumer/metadata.d.ts +0 -24
- package/dist/consumer/metadata.js +0 -64
- package/dist/consumer/offset-manager.d.ts +0 -22
- package/dist/consumer/offset-manager.js +0 -56
- package/dist/distributors/assignments-to-replicas.d.ts +0 -17
- package/dist/distributors/assignments-to-replicas.js +0 -60
- package/dist/distributors/assignments-to-replicas.test.d.ts +0 -1
- package/dist/distributors/assignments-to-replicas.test.js +0 -40
- package/dist/distributors/messages-to-topic-partition-leaders.d.ts +0 -17
- package/dist/distributors/messages-to-topic-partition-leaders.js +0 -15
- package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts +0 -1
- package/dist/distributors/messages-to-topic-partition-leaders.test.js +0 -30
- package/dist/examples/src/replicator.js +0 -34
- package/dist/examples/src/utils/json.js +0 -5
- package/dist/index.d.ts +0 -3
- package/dist/index.js +0 -19
- package/dist/metadata.d.ts +0 -24
- package/dist/metadata.js +0 -89
- package/dist/producer/producer.d.ts +0 -19
- package/dist/producer/producer.js +0 -111
- package/dist/request-handler.d.ts +0 -16
- package/dist/request-handler.js +0 -67
- package/dist/request-handler.test.d.ts +0 -1
- package/dist/request-handler.test.js +0 -340
- package/dist/src/api/api-versions.js +0 -18
- package/dist/src/api/create-topics.js +0 -46
- package/dist/src/api/delete-topics.js +0 -26
- package/dist/src/api/fetch.js +0 -95
- package/dist/src/api/find-coordinator.js +0 -34
- package/dist/src/api/heartbeat.js +0 -22
- package/dist/src/api/index.js +0 -38
- package/dist/src/api/init-producer-id.js +0 -24
- package/dist/src/api/join-group.js +0 -48
- package/dist/src/api/leave-group.js +0 -30
- package/dist/src/api/list-offsets.js +0 -39
- package/dist/src/api/metadata.js +0 -47
- package/dist/src/api/offset-commit.js +0 -39
- package/dist/src/api/offset-fetch.js +0 -44
- package/dist/src/api/produce.js +0 -119
- package/dist/src/api/sync-group.js +0 -31
- package/dist/src/broker.js +0 -35
- package/dist/src/connection.js +0 -21
- package/dist/src/consumer/consumer-group.js +0 -131
- package/dist/src/consumer/consumer.js +0 -103
- package/dist/src/consumer/metadata.js +0 -52
- package/dist/src/consumer/offset-manager.js +0 -23
- package/dist/src/index.js +0 -19
- package/dist/src/producer/producer.js +0 -84
- package/dist/src/request-handler.js +0 -57
- package/dist/src/request-handler.test.js +0 -321
- package/dist/src/types.js +0 -2
- package/dist/src/utils/api.js +0 -5
- package/dist/src/utils/decoder.js +0 -161
- package/dist/src/utils/encoder.js +0 -137
- package/dist/src/utils/error.js +0 -10
- package/dist/types.d.ts +0 -9
- package/dist/types.js +0 -2
- package/dist/utils/api.d.ts +0 -9
- package/dist/utils/api.js +0 -5
- package/dist/utils/debug.d.ts +0 -2
- package/dist/utils/debug.js +0 -11
- package/dist/utils/decoder.d.ts +0 -29
- package/dist/utils/decoder.js +0 -147
- package/dist/utils/delay.d.ts +0 -1
- package/dist/utils/delay.js +0 -5
- package/dist/utils/encoder.d.ts +0 -28
- package/dist/utils/encoder.js +0 -122
- package/dist/utils/error.d.ts +0 -11
- package/dist/utils/error.js +0 -27
- package/dist/utils/memo.d.ts +0 -1
- package/dist/utils/memo.js +0 -16
- package/dist/utils/retrier.d.ts +0 -10
- package/dist/utils/retrier.js +0 -22
- package/dist/utils/tracer.d.ts +0 -1
- package/dist/utils/tracer.js +0 -26
- package/examples/node_modules/.package-lock.json +0 -22
package/src/api/api-versions.ts
CHANGED
package/src/api/create-topics.ts
CHANGED
package/src/api/delete-topics.ts
CHANGED
package/src/api/fetch.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { createApi } from
|
|
2
|
-
import { Decoder } from
|
|
3
|
-
import { KafkaTSApiError } from
|
|
1
|
+
import { createApi } from '../utils/api';
|
|
2
|
+
import { Decoder } from '../utils/decoder';
|
|
3
|
+
import { KafkaTSApiError } from '../utils/error';
|
|
4
4
|
|
|
5
5
|
export const enum IsolationLevel {
|
|
6
6
|
READ_UNCOMMITTED = 0,
|
package/src/api/heartbeat.ts
CHANGED
package/src/api/index.ts
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import { Api } from
|
|
2
|
-
import { API_VERSIONS } from
|
|
3
|
-
import { CREATE_TOPICS } from
|
|
4
|
-
import { DELETE_TOPICS } from
|
|
5
|
-
import { FETCH } from
|
|
6
|
-
import { FIND_COORDINATOR } from
|
|
7
|
-
import { HEARTBEAT } from
|
|
8
|
-
import { INIT_PRODUCER_ID } from
|
|
9
|
-
import { JOIN_GROUP } from
|
|
10
|
-
import { LEAVE_GROUP } from
|
|
11
|
-
import { LIST_OFFSETS } from
|
|
12
|
-
import { METADATA } from
|
|
13
|
-
import { OFFSET_COMMIT } from
|
|
14
|
-
import { OFFSET_FETCH } from
|
|
15
|
-
import { PRODUCE } from
|
|
16
|
-
import { SASL_AUTHENTICATE } from
|
|
17
|
-
import { SASL_HANDSHAKE } from
|
|
18
|
-
import { SYNC_GROUP } from
|
|
1
|
+
import { Api } from '../utils/api';
|
|
2
|
+
import { API_VERSIONS } from './api-versions';
|
|
3
|
+
import { CREATE_TOPICS } from './create-topics';
|
|
4
|
+
import { DELETE_TOPICS } from './delete-topics';
|
|
5
|
+
import { FETCH } from './fetch';
|
|
6
|
+
import { FIND_COORDINATOR } from './find-coordinator';
|
|
7
|
+
import { HEARTBEAT } from './heartbeat';
|
|
8
|
+
import { INIT_PRODUCER_ID } from './init-producer-id';
|
|
9
|
+
import { JOIN_GROUP } from './join-group';
|
|
10
|
+
import { LEAVE_GROUP } from './leave-group';
|
|
11
|
+
import { LIST_OFFSETS } from './list-offsets';
|
|
12
|
+
import { METADATA } from './metadata';
|
|
13
|
+
import { OFFSET_COMMIT } from './offset-commit';
|
|
14
|
+
import { OFFSET_FETCH } from './offset-fetch';
|
|
15
|
+
import { PRODUCE } from './produce';
|
|
16
|
+
import { SASL_AUTHENTICATE } from './sasl-authenticate';
|
|
17
|
+
import { SASL_HANDSHAKE } from './sasl-handshake';
|
|
18
|
+
import { SYNC_GROUP } from './sync-group';
|
|
19
19
|
|
|
20
20
|
export const API = {
|
|
21
21
|
API_VERSIONS,
|
package/src/api/join-group.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { createApi } from
|
|
2
|
-
import { Encoder } from
|
|
3
|
-
import { KafkaTSApiError } from
|
|
1
|
+
import { createApi } from '../utils/api';
|
|
2
|
+
import { Encoder } from '../utils/encoder';
|
|
3
|
+
import { KafkaTSApiError } from '../utils/error';
|
|
4
4
|
|
|
5
5
|
export const JOIN_GROUP = createApi({
|
|
6
6
|
apiKey: 11,
|
package/src/api/leave-group.ts
CHANGED
package/src/api/list-offsets.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { createApi } from
|
|
2
|
-
import { KafkaTSApiError } from
|
|
3
|
-
import { IsolationLevel } from
|
|
1
|
+
import { createApi } from '../utils/api';
|
|
2
|
+
import { KafkaTSApiError } from '../utils/error';
|
|
3
|
+
import { IsolationLevel } from './fetch';
|
|
4
4
|
|
|
5
5
|
export const LIST_OFFSETS = createApi({
|
|
6
6
|
apiKey: 2,
|
package/src/api/metadata.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { createApi } from
|
|
2
|
-
import { KafkaTSApiError } from
|
|
1
|
+
import { createApi } from '../utils/api';
|
|
2
|
+
import { KafkaTSApiError } from '../utils/error';
|
|
3
3
|
|
|
4
|
-
export type Metadata = ReturnType<(typeof METADATA)[
|
|
4
|
+
export type Metadata = ReturnType<(typeof METADATA)['response']>;
|
|
5
5
|
|
|
6
6
|
export const METADATA = createApi({
|
|
7
7
|
apiKey: 3,
|
package/src/api/offset-commit.ts
CHANGED
package/src/api/offset-fetch.ts
CHANGED
package/src/api/produce.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { createApi } from
|
|
2
|
-
import { Encoder } from
|
|
3
|
-
import { KafkaTSApiError } from
|
|
1
|
+
import { createApi } from '../utils/api.js';
|
|
2
|
+
import { Encoder } from '../utils/encoder.js';
|
|
3
|
+
import { KafkaTSApiError } from '../utils/error.js';
|
|
4
4
|
|
|
5
5
|
export const PRODUCE = createApi({
|
|
6
6
|
apiKey: 0,
|
package/src/api/sync-group.ts
CHANGED
package/src/broker.ts
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
|
-
import { TcpSocketConnectOpts } from
|
|
2
|
-
import { TLSSocketOptions } from
|
|
3
|
-
import { API } from
|
|
4
|
-
import { Connection, SendRequest } from
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
1
|
+
import { TcpSocketConnectOpts } from 'net';
|
|
2
|
+
import { TLSSocketOptions } from 'tls';
|
|
3
|
+
import { API } from './api';
|
|
4
|
+
import { Connection, SendRequest } from './connection';
|
|
5
|
+
import { base64Decode, base64Encode, generateNonce, hash, hmac, saltPassword, xor } from './utils/crypto';
|
|
6
|
+
import { KafkaTSError } from './utils/error';
|
|
7
|
+
import { memo } from './utils/memo';
|
|
7
8
|
|
|
8
|
-
export type SASLOptions = {
|
|
9
|
+
export type SASLOptions = {
|
|
10
|
+
mechanism: 'PLAIN' | 'SCRAM-SHA-256';
|
|
11
|
+
username: string;
|
|
12
|
+
password: string;
|
|
13
|
+
};
|
|
9
14
|
|
|
10
15
|
type BrokerOptions = {
|
|
11
16
|
clientId: string | null;
|
|
@@ -64,11 +69,78 @@ export class Broker {
|
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
private async saslAuthenticate() {
|
|
67
|
-
if (this.options.sasl
|
|
72
|
+
if (!this.options.sasl) {
|
|
68
73
|
return;
|
|
69
74
|
}
|
|
70
|
-
|
|
71
|
-
const
|
|
72
|
-
|
|
75
|
+
|
|
76
|
+
const { mechanism } = this.options.sasl;
|
|
77
|
+
const authenticate = { PLAIN: plainProvider, 'SCRAM-SHA-256': scramSha256Provider }[mechanism as string];
|
|
78
|
+
if (!authenticate) {
|
|
79
|
+
throw new KafkaTSError(`SASL mechanism ${mechanism} is not supported`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
await authenticate({
|
|
83
|
+
...this.options.sasl,
|
|
84
|
+
sendRequest: this.sendRequest.bind(this),
|
|
85
|
+
});
|
|
73
86
|
}
|
|
74
87
|
}
|
|
88
|
+
|
|
89
|
+
const plainProvider = async ({
|
|
90
|
+
username,
|
|
91
|
+
password,
|
|
92
|
+
sendRequest,
|
|
93
|
+
}: {
|
|
94
|
+
username: string;
|
|
95
|
+
password: string;
|
|
96
|
+
sendRequest: SendRequest;
|
|
97
|
+
}) => {
|
|
98
|
+
const authBytes = [null, username, password].join('\u0000');
|
|
99
|
+
await sendRequest(API.SASL_AUTHENTICATE, { authBytes: Buffer.from(authBytes) });
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
const scramSha256Provider = async ({
|
|
103
|
+
username,
|
|
104
|
+
password,
|
|
105
|
+
sendRequest,
|
|
106
|
+
}: {
|
|
107
|
+
username: string;
|
|
108
|
+
password: string;
|
|
109
|
+
sendRequest: SendRequest;
|
|
110
|
+
}) => {
|
|
111
|
+
const nonce = generateNonce();
|
|
112
|
+
const firstMessage = `n=${username},r=${nonce}`;
|
|
113
|
+
|
|
114
|
+
const { authBytes } = await sendRequest(API.SASL_AUTHENTICATE, { authBytes: Buffer.from(`n,,${firstMessage}`) });
|
|
115
|
+
if (!authBytes) {
|
|
116
|
+
throw new KafkaTSError('No auth response');
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const response = Object.fromEntries(
|
|
120
|
+
authBytes
|
|
121
|
+
.toString()
|
|
122
|
+
.split(',')
|
|
123
|
+
.map((pair) => pair.split('=')),
|
|
124
|
+
) as { r: string; s: string; i: string };
|
|
125
|
+
|
|
126
|
+
const rnonce = response.r;
|
|
127
|
+
if (!rnonce.startsWith(nonce)) {
|
|
128
|
+
throw new KafkaTSError('Invalid nonce');
|
|
129
|
+
}
|
|
130
|
+
const iterations = parseInt(response.i);
|
|
131
|
+
const salt = base64Decode(response.s);
|
|
132
|
+
|
|
133
|
+
const saltedPassword = await saltPassword(password, salt, iterations, 32, 'sha256');
|
|
134
|
+
const clientKey = hmac(saltedPassword, 'Client Key');
|
|
135
|
+
const clientKeyHash = hash(clientKey);
|
|
136
|
+
|
|
137
|
+
let finalMessage = `c=${base64Encode('n,,')},r=${rnonce}`;
|
|
138
|
+
|
|
139
|
+
const fullMessage = `${firstMessage},${authBytes.toString()},${finalMessage}`;
|
|
140
|
+
const clientSignature = hmac(clientKeyHash, fullMessage);
|
|
141
|
+
const clientProof = base64Encode(xor(clientKey, clientSignature));
|
|
142
|
+
|
|
143
|
+
finalMessage += `,p=${clientProof}`;
|
|
144
|
+
|
|
145
|
+
await sendRequest(API.SASL_AUTHENTICATE, { authBytes: Buffer.from(finalMessage) });
|
|
146
|
+
};
|
package/src/client.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { TcpSocketConnectOpts } from
|
|
2
|
-
import { TLSSocketOptions } from
|
|
3
|
-
import { SASLOptions } from
|
|
4
|
-
import { Cluster } from
|
|
5
|
-
import { Consumer, ConsumerOptions } from
|
|
6
|
-
import { Producer, ProducerOptions } from
|
|
1
|
+
import { TcpSocketConnectOpts } from 'net';
|
|
2
|
+
import { TLSSocketOptions } from 'tls';
|
|
3
|
+
import { SASLOptions } from './broker';
|
|
4
|
+
import { Cluster } from './cluster';
|
|
5
|
+
import { Consumer, ConsumerOptions } from './consumer/consumer';
|
|
6
|
+
import { Producer, ProducerOptions } from './producer/producer';
|
|
7
7
|
|
|
8
8
|
type ClientOptions = {
|
|
9
9
|
clientId?: string | null;
|