@platformatic/kafka 0.4.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/apis/definitions.js +0 -1
- package/dist/clients/admin/admin.js +10 -10
- package/dist/clients/base/base.d.ts +9 -2
- package/dist/clients/base/base.js +20 -9
- package/dist/clients/callbacks.d.ts +1 -0
- package/dist/clients/callbacks.js +3 -2
- package/dist/clients/consumer/consumer.js +102 -86
- package/dist/clients/consumer/messages-stream.d.ts +1 -0
- package/dist/clients/consumer/messages-stream.js +81 -47
- package/dist/clients/consumer/options.d.ts +3 -0
- package/dist/clients/consumer/options.js +1 -0
- package/dist/clients/consumer/types.d.ts +7 -2
- package/dist/clients/producer/producer.js +33 -22
- package/dist/diagnostic.d.ts +46 -0
- package/dist/diagnostic.js +37 -0
- package/dist/errors.js +0 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/network/connection-pool.d.ts +2 -1
- package/dist/network/connection-pool.js +36 -25
- package/dist/network/connection.d.ts +2 -2
- package/dist/network/connection.js +135 -81
- package/dist/protocol/compression.js +8 -8
- package/dist/protocol/reader.js +0 -1
- package/dist/protocol/records.js +0 -3
- package/dist/protocol/sasl/scram-sha.d.ts +10 -3
- package/dist/protocol/sasl/scram-sha.js +8 -2
- package/dist/symbols.d.ts +1 -0
- package/dist/symbols.js +2 -0
- package/package.json +15 -13
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import EventEmitter from 'node:events';
|
|
2
2
|
import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../clients/callbacks.js";
|
|
3
|
+
import { connectionsPoolGetsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
|
|
3
4
|
import { MultipleErrors } from "../errors.js";
|
|
4
5
|
import { Connection, ConnectionStatuses } from "./connection.js";
|
|
6
|
+
let currentInstance = 0;
|
|
5
7
|
export class ConnectionPool extends EventEmitter {
|
|
8
|
+
#instanceId;
|
|
6
9
|
#clientId;
|
|
7
10
|
// @ts-ignore This is used just for debugging
|
|
8
11
|
#ownerId;
|
|
@@ -10,17 +13,47 @@ export class ConnectionPool extends EventEmitter {
|
|
|
10
13
|
#connectionOptions;
|
|
11
14
|
constructor(clientId, connectionOptions = {}) {
|
|
12
15
|
super();
|
|
16
|
+
this.#instanceId = currentInstance++;
|
|
13
17
|
this.#clientId = clientId;
|
|
14
18
|
this.#ownerId = connectionOptions.ownerId;
|
|
15
19
|
this.#connections = new Map();
|
|
16
20
|
this.#connectionOptions = connectionOptions;
|
|
21
|
+
notifyCreation('connection-pool', this);
|
|
22
|
+
}
|
|
23
|
+
get instanceId() {
|
|
24
|
+
return this.#instanceId;
|
|
17
25
|
}
|
|
18
26
|
get(broker, callback) {
|
|
19
|
-
const key = `${broker.host}:${broker.port}`;
|
|
20
|
-
const existing = this.#connections.get(key);
|
|
21
27
|
if (!callback) {
|
|
22
28
|
callback = createPromisifiedCallback();
|
|
23
29
|
}
|
|
30
|
+
connectionsPoolGetsChannel.traceCallback(this.#get, 1, createDiagnosticContext({ connectionPool: this, broker, operation: 'get' }), this, broker, callback);
|
|
31
|
+
return callback[kCallbackPromise];
|
|
32
|
+
}
|
|
33
|
+
getFirstAvailable(brokers, callback) {
|
|
34
|
+
if (!callback) {
|
|
35
|
+
callback = createPromisifiedCallback();
|
|
36
|
+
}
|
|
37
|
+
connectionsPoolGetsChannel.traceCallback(this.#getFirstAvailable, 3, createDiagnosticContext({ connectionPool: this, brokers, operation: 'getFirstAvailable' }), this, brokers, 0, [], callback);
|
|
38
|
+
return callback[kCallbackPromise];
|
|
39
|
+
}
|
|
40
|
+
close(callback) {
|
|
41
|
+
if (!callback) {
|
|
42
|
+
callback = createPromisifiedCallback();
|
|
43
|
+
}
|
|
44
|
+
if (this.#connections.size === 0) {
|
|
45
|
+
callback(null);
|
|
46
|
+
return callback[kCallbackPromise];
|
|
47
|
+
}
|
|
48
|
+
runConcurrentCallbacks('Closing connections failed.', this.#connections, ([key, connection], cb) => {
|
|
49
|
+
connection.close(cb);
|
|
50
|
+
this.#connections.delete(key);
|
|
51
|
+
}, error => callback(error));
|
|
52
|
+
return callback[kCallbackPromise];
|
|
53
|
+
}
|
|
54
|
+
#get(broker, callback) {
|
|
55
|
+
const key = `${broker.host}:${broker.port}`;
|
|
56
|
+
const existing = this.#connections.get(key);
|
|
24
57
|
if (existing) {
|
|
25
58
|
if (existing.status !== ConnectionStatuses.CONNECTED) {
|
|
26
59
|
existing.ready(error => {
|
|
@@ -34,7 +67,7 @@ export class ConnectionPool extends EventEmitter {
|
|
|
34
67
|
else {
|
|
35
68
|
callback(null, existing);
|
|
36
69
|
}
|
|
37
|
-
return
|
|
70
|
+
return;
|
|
38
71
|
}
|
|
39
72
|
const connection = new Connection(this.#clientId, this.#connectionOptions);
|
|
40
73
|
this.#connections.set(key, connection);
|
|
@@ -61,28 +94,6 @@ export class ConnectionPool extends EventEmitter {
|
|
|
61
94
|
connection.on('drain', () => {
|
|
62
95
|
this.emit('drain', eventPayload);
|
|
63
96
|
});
|
|
64
|
-
return callback[kCallbackPromise];
|
|
65
|
-
}
|
|
66
|
-
getFirstAvailable(brokers, callback, current = 0, errors = []) {
|
|
67
|
-
if (!callback) {
|
|
68
|
-
callback = createPromisifiedCallback();
|
|
69
|
-
}
|
|
70
|
-
this.#getFirstAvailable(brokers, current, errors, callback);
|
|
71
|
-
return callback[kCallbackPromise];
|
|
72
|
-
}
|
|
73
|
-
close(callback) {
|
|
74
|
-
if (!callback) {
|
|
75
|
-
callback = createPromisifiedCallback();
|
|
76
|
-
}
|
|
77
|
-
if (this.#connections.size === 0) {
|
|
78
|
-
callback(null);
|
|
79
|
-
return callback[kCallbackPromise];
|
|
80
|
-
}
|
|
81
|
-
runConcurrentCallbacks('Closing connections failed.', this.#connections, ([key, connection], cb) => {
|
|
82
|
-
connection.close(cb);
|
|
83
|
-
this.#connections.delete(key);
|
|
84
|
-
}, error => callback(error));
|
|
85
|
-
return callback[kCallbackPromise];
|
|
86
97
|
}
|
|
87
98
|
#getFirstAvailable(brokers, current = 0, errors = [], callback) {
|
|
88
99
|
this.get(brokers[current], (error, connection) => {
|
|
@@ -23,6 +23,7 @@ export interface Request {
|
|
|
23
23
|
payload: () => Writer;
|
|
24
24
|
parser: ResponseParser<unknown>;
|
|
25
25
|
callback: Callback<any>;
|
|
26
|
+
diagnostic: Record<string, unknown>;
|
|
26
27
|
}
|
|
27
28
|
export declare const ConnectionStatuses: {
|
|
28
29
|
readonly NONE: "none";
|
|
@@ -35,11 +36,10 @@ export declare const ConnectionStatuses: {
|
|
|
35
36
|
export type ConnectionStatus = keyof typeof ConnectionStatuses;
|
|
36
37
|
export type ConnectionStatusValue = (typeof ConnectionStatuses)[keyof typeof ConnectionStatuses];
|
|
37
38
|
export declare const defaultOptions: ConnectionOptions;
|
|
38
|
-
export declare function noResponseCallback(..._: any[]): void;
|
|
39
|
-
export declare namespace noResponseCallback { }
|
|
40
39
|
export declare class Connection extends EventEmitter {
|
|
41
40
|
#private;
|
|
42
41
|
constructor(clientId?: string, options?: ConnectionOptions);
|
|
42
|
+
get instanceId(): number;
|
|
43
43
|
get status(): ConnectionStatusValue;
|
|
44
44
|
get socket(): Socket;
|
|
45
45
|
connect(host: string, port: number, callback?: CallbackWithPromise<void>): void | Promise<void>;
|
|
@@ -3,6 +3,7 @@ import EventEmitter from 'node:events';
|
|
|
3
3
|
import { createConnection } from 'node:net';
|
|
4
4
|
import { connect as createTLSConnection } from 'node:tls';
|
|
5
5
|
import { createPromisifiedCallback, kCallbackPromise } from "../clients/callbacks.js";
|
|
6
|
+
import { connectionsApiChannel, connectionsConnectsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
|
|
6
7
|
import { NetworkError, TimeoutError, UnexpectedCorrelationIdError } from "../errors.js";
|
|
7
8
|
import { protocolAPIsById } from "../protocol/apis.js";
|
|
8
9
|
import { EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE, INT32_SIZE } from "../protocol/definitions.js";
|
|
@@ -22,13 +23,11 @@ export const defaultOptions = {
|
|
|
22
23
|
connectTimeout: 5000,
|
|
23
24
|
maxInflights: 5
|
|
24
25
|
};
|
|
25
|
-
|
|
26
|
-
/* c8 ignore next */
|
|
27
|
-
export function noResponseCallback(..._) { }
|
|
28
|
-
noResponseCallback[kNoResponse] = true;
|
|
26
|
+
let currentInstance = 0;
|
|
29
27
|
export class Connection extends EventEmitter {
|
|
30
28
|
#options;
|
|
31
29
|
#status;
|
|
30
|
+
#instanceId;
|
|
32
31
|
#clientId;
|
|
33
32
|
// @ts-ignore This is used just for debugging
|
|
34
33
|
#ownerId;
|
|
@@ -44,6 +43,7 @@ export class Connection extends EventEmitter {
|
|
|
44
43
|
constructor(clientId, options = {}) {
|
|
45
44
|
super();
|
|
46
45
|
this.setMaxListeners(0);
|
|
46
|
+
this.#instanceId = currentInstance++;
|
|
47
47
|
this.#options = Object.assign({}, defaultOptions, options);
|
|
48
48
|
this.#status = ConnectionStatuses.NONE;
|
|
49
49
|
this.#clientId = clientId;
|
|
@@ -56,6 +56,10 @@ export class Connection extends EventEmitter {
|
|
|
56
56
|
this.#responseBuffer = new DynamicBuffer();
|
|
57
57
|
this.#responseReader = new Reader(this.#responseBuffer);
|
|
58
58
|
this.#socketMustBeDrained = false;
|
|
59
|
+
notifyCreation('connection', this);
|
|
60
|
+
}
|
|
61
|
+
get instanceId() {
|
|
62
|
+
return this.#instanceId;
|
|
59
63
|
}
|
|
60
64
|
get status() {
|
|
61
65
|
return this.#status;
|
|
@@ -67,49 +71,72 @@ export class Connection extends EventEmitter {
|
|
|
67
71
|
if (!callback) {
|
|
68
72
|
callback = createPromisifiedCallback();
|
|
69
73
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
const diagnosticContext = createDiagnosticContext({ connection: this, operation: 'connect', host, port });
|
|
75
|
+
connectionsConnectsChannel.start.publish(diagnosticContext);
|
|
76
|
+
try {
|
|
77
|
+
if (this.#status === ConnectionStatuses.CONNECTED) {
|
|
78
|
+
callback(null);
|
|
79
|
+
return callback[kCallbackPromise];
|
|
80
|
+
}
|
|
81
|
+
this.ready(callback);
|
|
82
|
+
if (this.#status === ConnectionStatuses.CONNECTING) {
|
|
83
|
+
return callback[kCallbackPromise];
|
|
84
|
+
}
|
|
85
|
+
this.#status = ConnectionStatuses.CONNECTING;
|
|
86
|
+
const connectionOptions = {
|
|
87
|
+
timeout: this.#options.connectTimeout
|
|
88
|
+
};
|
|
89
|
+
const connectionTimeoutHandler = () => {
|
|
90
|
+
const error = new TimeoutError(`Connection to ${host}:${port} timed out.`);
|
|
91
|
+
diagnosticContext.error = error;
|
|
92
|
+
this.#socket.destroy();
|
|
93
|
+
this.#status = ConnectionStatuses.ERROR;
|
|
94
|
+
connectionsConnectsChannel.error.publish(diagnosticContext);
|
|
95
|
+
connectionsConnectsChannel.asyncStart.publish(diagnosticContext);
|
|
96
|
+
this.emit('timeout', error);
|
|
97
|
+
this.emit('error', error);
|
|
98
|
+
connectionsConnectsChannel.asyncEnd.publish(diagnosticContext);
|
|
99
|
+
};
|
|
100
|
+
const connectionErrorHandler = (e) => {
|
|
101
|
+
const error = new NetworkError(`Connection to ${host}:${port} failed.`, { cause: e });
|
|
102
|
+
diagnosticContext.error = error;
|
|
103
|
+
this.#status = ConnectionStatuses.ERROR;
|
|
104
|
+
connectionsConnectsChannel.error.publish(diagnosticContext);
|
|
105
|
+
connectionsConnectsChannel.asyncStart.publish(diagnosticContext);
|
|
106
|
+
this.emit('error', error);
|
|
107
|
+
connectionsConnectsChannel.asyncEnd.publish(diagnosticContext);
|
|
108
|
+
};
|
|
109
|
+
this.emit('connecting');
|
|
110
|
+
/* c8 ignore next 3 - TLS connection is not tested but we rely on Node.js tests */
|
|
111
|
+
this.#socket = this.#options.tls
|
|
112
|
+
? createTLSConnection(port, host, { ...this.#options.tls, ...connectionOptions })
|
|
113
|
+
: createConnection({ ...connectionOptions, port, host });
|
|
114
|
+
this.#socket.setNoDelay(true);
|
|
115
|
+
this.#socket.once('connect', () => {
|
|
116
|
+
this.#socket.removeListener('timeout', connectionTimeoutHandler);
|
|
117
|
+
this.#socket.removeListener('error', connectionErrorHandler);
|
|
118
|
+
this.#socket.on('error', this.#onError.bind(this));
|
|
119
|
+
this.#socket.on('data', this.#onData.bind(this));
|
|
120
|
+
this.#socket.on('drain', this.#onDrain.bind(this));
|
|
121
|
+
this.#socket.on('close', this.#onClose.bind(this));
|
|
122
|
+
this.#socket.setTimeout(0);
|
|
123
|
+
this.#status = ConnectionStatuses.CONNECTED;
|
|
124
|
+
connectionsConnectsChannel.asyncStart.publish(diagnosticContext);
|
|
125
|
+
this.emit('connect');
|
|
126
|
+
connectionsConnectsChannel.asyncEnd.publish(diagnosticContext);
|
|
127
|
+
});
|
|
128
|
+
this.#socket.once('timeout', connectionTimeoutHandler);
|
|
129
|
+
this.#socket.once('error', connectionErrorHandler);
|
|
77
130
|
}
|
|
78
|
-
|
|
79
|
-
const connectionOptions = {
|
|
80
|
-
timeout: this.#options.connectTimeout
|
|
81
|
-
};
|
|
82
|
-
const connectionTimeoutHandler = () => {
|
|
83
|
-
const error = new TimeoutError(`Connection to ${host}:${port} timed out.`);
|
|
84
|
-
this.#socket.destroy();
|
|
85
|
-
this.#status = ConnectionStatuses.ERROR;
|
|
86
|
-
this.emit('timeout', error);
|
|
87
|
-
this.emit('error', error);
|
|
88
|
-
};
|
|
89
|
-
const connectionErrorHandler = (e) => {
|
|
90
|
-
const error = new NetworkError(`Connection to ${host}:${port} failed.`, { cause: e });
|
|
131
|
+
catch (error) {
|
|
91
132
|
this.#status = ConnectionStatuses.ERROR;
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
this.#socket.setNoDelay(true);
|
|
100
|
-
this.#socket.once('connect', () => {
|
|
101
|
-
this.#socket.removeListener('timeout', connectionTimeoutHandler);
|
|
102
|
-
this.#socket.removeListener('error', connectionErrorHandler);
|
|
103
|
-
this.#socket.on('error', this.#onError.bind(this));
|
|
104
|
-
this.#socket.on('data', this.#onData.bind(this));
|
|
105
|
-
this.#socket.on('drain', this.#onDrain.bind(this));
|
|
106
|
-
this.#socket.on('close', this.#onClose.bind(this));
|
|
107
|
-
this.#socket.setTimeout(0);
|
|
108
|
-
this.#status = ConnectionStatuses.CONNECTED;
|
|
109
|
-
this.emit('connect');
|
|
110
|
-
});
|
|
111
|
-
this.#socket.once('timeout', connectionTimeoutHandler);
|
|
112
|
-
this.#socket.once('error', connectionErrorHandler);
|
|
133
|
+
diagnosticContext.error = error;
|
|
134
|
+
connectionsConnectsChannel.error.publish(diagnosticContext);
|
|
135
|
+
throw error;
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
connectionsConnectsChannel.end.publish(diagnosticContext);
|
|
139
|
+
}
|
|
113
140
|
return callback[kCallbackPromise];
|
|
114
141
|
}
|
|
115
142
|
ready(callback) {
|
|
@@ -169,7 +196,14 @@ export class Connection extends EventEmitter {
|
|
|
169
196
|
hasResponseHeaderTaggedFields,
|
|
170
197
|
parser: responseParser,
|
|
171
198
|
payload,
|
|
172
|
-
callback: fastQueueCallback
|
|
199
|
+
callback: fastQueueCallback,
|
|
200
|
+
diagnostic: createDiagnosticContext({
|
|
201
|
+
connection: this,
|
|
202
|
+
operation: 'send',
|
|
203
|
+
apiKey,
|
|
204
|
+
apiVersion,
|
|
205
|
+
correlationId
|
|
206
|
+
})
|
|
173
207
|
};
|
|
174
208
|
if (this.#socketMustBeDrained) {
|
|
175
209
|
this.#afterDrainRequests.push(request);
|
|
@@ -187,44 +221,56 @@ export class Connection extends EventEmitter {
|
|
|
187
221
|
client_id => NULLABLE_STRING
|
|
188
222
|
*/
|
|
189
223
|
#sendRequest(request) {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
.
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
writer.appendFrom(payload);
|
|
206
|
-
writer.prependLength();
|
|
207
|
-
// Write the header
|
|
208
|
-
this.#socket.cork();
|
|
209
|
-
if (!payload.context.noResponse) {
|
|
210
|
-
this.#inflightRequests.set(correlationId, request);
|
|
211
|
-
}
|
|
212
|
-
/* c8 ignore next */
|
|
213
|
-
loggers.protocol({ apiKey: protocolAPIsById[apiKey], correlationId, request }, 'Sending request.');
|
|
214
|
-
for (const buf of writer.buffers) {
|
|
215
|
-
if (!this.#socket.write(buf)) {
|
|
216
|
-
canWrite = false;
|
|
224
|
+
connectionsApiChannel.start.publish(request.diagnostic);
|
|
225
|
+
try {
|
|
226
|
+
if (this.#status !== ConnectionStatuses.CONNECTED) {
|
|
227
|
+
request.callback(new NetworkError('Connection closed'), undefined);
|
|
228
|
+
return false;
|
|
229
|
+
}
|
|
230
|
+
let canWrite = true;
|
|
231
|
+
const { correlationId, apiKey, apiVersion, payload: payloadFn, hasRequestHeaderTaggedFields } = request;
|
|
232
|
+
const writer = Writer.create()
|
|
233
|
+
.appendInt16(apiKey)
|
|
234
|
+
.appendInt16(apiVersion)
|
|
235
|
+
.appendInt32(correlationId)
|
|
236
|
+
.appendString(this.#clientId, false);
|
|
237
|
+
if (hasRequestHeaderTaggedFields) {
|
|
238
|
+
writer.appendTaggedFields();
|
|
217
239
|
}
|
|
240
|
+
const payload = payloadFn();
|
|
241
|
+
writer.appendFrom(payload);
|
|
242
|
+
writer.prependLength();
|
|
243
|
+
// Write the header
|
|
244
|
+
this.#socket.cork();
|
|
245
|
+
if (!payload.context.noResponse) {
|
|
246
|
+
this.#inflightRequests.set(correlationId, request);
|
|
247
|
+
}
|
|
248
|
+
loggers.protocol({ apiKey: protocolAPIsById[apiKey], correlationId, request }, 'Sending request.');
|
|
249
|
+
for (const buf of writer.buffers) {
|
|
250
|
+
if (!this.#socket.write(buf)) {
|
|
251
|
+
canWrite = false;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (!canWrite) {
|
|
255
|
+
this.#socketMustBeDrained = true;
|
|
256
|
+
}
|
|
257
|
+
this.#socket.uncork();
|
|
258
|
+
if (payload.context.noResponse) {
|
|
259
|
+
request.callback(null, canWrite);
|
|
260
|
+
}
|
|
261
|
+
// debugDump(Date.now() % 100000, 'send', { owner: this.#ownerId, apiKey: protocolAPIsById[apiKey], correlationId })
|
|
262
|
+
return canWrite;
|
|
218
263
|
}
|
|
219
|
-
|
|
220
|
-
|
|
264
|
+
catch (error) {
|
|
265
|
+
request.diagnostic.error = error;
|
|
266
|
+
connectionsApiChannel.error.publish(request.diagnostic);
|
|
267
|
+
connectionsApiChannel.end.publish(request.diagnostic);
|
|
268
|
+
throw error;
|
|
269
|
+
/* c8 ignore next 3 - C8 does not detect these as covered */
|
|
221
270
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
request.callback(null, canWrite);
|
|
271
|
+
finally {
|
|
272
|
+
connectionsApiChannel.end.publish(request.diagnostic);
|
|
225
273
|
}
|
|
226
|
-
// debugDump(Date.now() % 100000, 'send', { owner: this.#ownerId, apiKey: protocolAPIsById[apiKey], correlationId })
|
|
227
|
-
return canWrite;
|
|
228
274
|
}
|
|
229
275
|
/*
|
|
230
276
|
Response Header v1 => correlation_id TAG_BUFFER
|
|
@@ -281,9 +327,17 @@ export class Connection extends EventEmitter {
|
|
|
281
327
|
// apiKey: protocolAPIsById[apiKey],
|
|
282
328
|
// correlationId
|
|
283
329
|
// })
|
|
284
|
-
/* c8 ignore next */
|
|
285
330
|
loggers.protocol({ apiKey: protocolAPIsById[apiKey], correlationId, request }, 'Received response.');
|
|
331
|
+
if (responseError) {
|
|
332
|
+
request.diagnostic.error = responseError;
|
|
333
|
+
connectionsApiChannel.error.publish(request.diagnostic);
|
|
334
|
+
}
|
|
335
|
+
else {
|
|
336
|
+
request.diagnostic.result = deserialized;
|
|
337
|
+
}
|
|
338
|
+
connectionsApiChannel.asyncStart.publish(request.diagnostic);
|
|
286
339
|
callback(responseError, deserialized);
|
|
340
|
+
connectionsApiChannel.asyncStart.publish(request.diagnostic);
|
|
287
341
|
}
|
|
288
342
|
}
|
|
289
343
|
#onDrain() {
|
|
@@ -17,7 +17,7 @@ function loadSnappy() {
|
|
|
17
17
|
const snappy = require('snappy');
|
|
18
18
|
snappyCompressSync = snappy.compressSync;
|
|
19
19
|
snappyDecompressSync = snappy.uncompressSync;
|
|
20
|
-
/* c8 ignore next 5 */
|
|
20
|
+
/* c8 ignore next 5 - In tests snappy is always available */
|
|
21
21
|
}
|
|
22
22
|
catch (e) {
|
|
23
23
|
throw new UnsupportedCompressionError('Cannot load snappy module, which is an optionalDependency. Please check your local installation.');
|
|
@@ -28,7 +28,7 @@ function loadLZ4() {
|
|
|
28
28
|
const lz4 = require('lz4-napi');
|
|
29
29
|
lz4CompressSync = lz4.compressSync;
|
|
30
30
|
lz4DecompressSync = lz4.uncompressSync;
|
|
31
|
-
/* c8 ignore next 5 */
|
|
31
|
+
/* c8 ignore next 5 - In tests lz4-napi is always available */
|
|
32
32
|
}
|
|
33
33
|
catch (e) {
|
|
34
34
|
throw new UnsupportedCompressionError('Cannot load lz4-napi module, which is an optionalDependency. Please check your local installation.');
|
|
@@ -58,14 +58,14 @@ export const compressionsAlgorithms = {
|
|
|
58
58
|
},
|
|
59
59
|
snappy: {
|
|
60
60
|
compressSync(data) {
|
|
61
|
-
/* c8 ignore next 4 */
|
|
61
|
+
/* c8 ignore next 4 - In tests snappy is always available */
|
|
62
62
|
if (!snappyCompressSync) {
|
|
63
63
|
loadSnappy();
|
|
64
64
|
}
|
|
65
65
|
return snappyCompressSync(ensureBuffer(data));
|
|
66
66
|
},
|
|
67
67
|
decompressSync(data) {
|
|
68
|
-
/* c8 ignore next 4 */
|
|
68
|
+
/* c8 ignore next 4 - In tests snappy is always available */
|
|
69
69
|
if (!snappyDecompressSync) {
|
|
70
70
|
loadSnappy();
|
|
71
71
|
}
|
|
@@ -76,14 +76,14 @@ export const compressionsAlgorithms = {
|
|
|
76
76
|
},
|
|
77
77
|
lz4: {
|
|
78
78
|
compressSync(data) {
|
|
79
|
-
/* c8 ignore next 4 */
|
|
79
|
+
/* c8 ignore next 4 - In tests lz4-napi is always available */
|
|
80
80
|
if (!lz4CompressSync) {
|
|
81
81
|
loadLZ4();
|
|
82
82
|
}
|
|
83
83
|
return lz4CompressSync(ensureBuffer(data));
|
|
84
84
|
},
|
|
85
85
|
decompressSync(data) {
|
|
86
|
-
/* c8 ignore next 4 */
|
|
86
|
+
/* c8 ignore next 4 - In tests lz4-napi is always available */
|
|
87
87
|
if (!lz4DecompressSync) {
|
|
88
88
|
loadLZ4();
|
|
89
89
|
}
|
|
@@ -93,14 +93,14 @@ export const compressionsAlgorithms = {
|
|
|
93
93
|
available: true
|
|
94
94
|
},
|
|
95
95
|
zstd: {
|
|
96
|
-
/* c8 ignore next 7 */
|
|
96
|
+
/* c8 ignore next 7 - Tests are only run on Node.js versions that support zstd */
|
|
97
97
|
compressSync(data) {
|
|
98
98
|
if (!zstdCompressSync) {
|
|
99
99
|
throw new UnsupportedCompressionError('zstd is not supported in the current Node.js version');
|
|
100
100
|
}
|
|
101
101
|
return zstdCompressSync(ensureBuffer(data));
|
|
102
102
|
},
|
|
103
|
-
/* c8 ignore next 7 */
|
|
103
|
+
/* c8 ignore next 7 - Tests are only run on Node.js versions that support zstd */
|
|
104
104
|
decompressSync(data) {
|
|
105
105
|
if (!zstdCompressSync) {
|
|
106
106
|
throw new UnsupportedCompressionError('zstd is not supported in the current Node.js version');
|
package/dist/protocol/reader.js
CHANGED
package/dist/protocol/records.js
CHANGED
|
@@ -83,7 +83,6 @@ export function createRecordsBatch(messages, options = {}) {
|
|
|
83
83
|
attributes |= IS_TRANSACTIONAL;
|
|
84
84
|
}
|
|
85
85
|
// Set the compression, if any
|
|
86
|
-
/* c8 ignore next */
|
|
87
86
|
if ((options.compression ?? 'none') !== 'none') {
|
|
88
87
|
const algorithm = compressionsAlgorithms[options.compression];
|
|
89
88
|
if (!algorithm) {
|
|
@@ -100,9 +99,7 @@ export function createRecordsBatch(messages, options = {}) {
|
|
|
100
99
|
.appendInt32(messages.length - 1)
|
|
101
100
|
.appendInt64(BigInt(firstTimestamp))
|
|
102
101
|
.appendInt64(BigInt(maxTimestamp))
|
|
103
|
-
/* c8 ignore next */
|
|
104
102
|
.appendInt64(options.producerId ?? -1n)
|
|
105
|
-
/* c8 ignore next */
|
|
106
103
|
.appendInt16(options.producerEpoch ?? 0)
|
|
107
104
|
.appendInt32(firstSequence)
|
|
108
105
|
.appendInt32(messages.length) // Number of records
|
|
@@ -5,6 +5,12 @@ export interface ScramAlgorithmDefinition {
|
|
|
5
5
|
algorithm: string;
|
|
6
6
|
minIterations: number;
|
|
7
7
|
}
|
|
8
|
+
export interface ScramCryptoModule {
|
|
9
|
+
h: (definition: ScramAlgorithmDefinition, data: string | Buffer) => Buffer;
|
|
10
|
+
hi: (definition: ScramAlgorithmDefinition, password: string, salt: Buffer, iterations: number) => Buffer;
|
|
11
|
+
hmac: (definition: ScramAlgorithmDefinition, key: Buffer, data: string | Buffer) => Buffer;
|
|
12
|
+
xor: (a: Buffer, b: Buffer) => Buffer;
|
|
13
|
+
}
|
|
8
14
|
export declare const ScramAlgorithms: {
|
|
9
15
|
readonly 'SHA-256': {
|
|
10
16
|
readonly keyLength: 32;
|
|
@@ -21,8 +27,9 @@ export type ScramAlgorithm = keyof typeof ScramAlgorithms;
|
|
|
21
27
|
export declare function createNonce(): string;
|
|
22
28
|
export declare function sanitizeString(str: string): string;
|
|
23
29
|
export declare function parseParameters(data: Buffer): Record<string, string>;
|
|
24
|
-
export declare function h(definition: ScramAlgorithmDefinition, data: string | Buffer): Buffer
|
|
25
|
-
export declare function hi(definition: ScramAlgorithmDefinition, password: string, salt: Buffer, iterations: number): Buffer
|
|
30
|
+
export declare function h(definition: ScramAlgorithmDefinition, data: string | Buffer): Buffer;
|
|
31
|
+
export declare function hi(definition: ScramAlgorithmDefinition, password: string, salt: Buffer, iterations: number): Buffer;
|
|
26
32
|
export declare function hmac(definition: ScramAlgorithmDefinition, key: Buffer, data: string | Buffer): Buffer;
|
|
27
33
|
export declare function xor(a: Buffer, b: Buffer): Buffer;
|
|
28
|
-
export declare
|
|
34
|
+
export declare const defaultCrypto: ScramCryptoModule;
|
|
35
|
+
export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, username: string, password: string, crypto?: ScramCryptoModule): Promise<SaslAuthenticateResponse>;
|
|
@@ -50,8 +50,15 @@ export function xor(a, b) {
|
|
|
50
50
|
}
|
|
51
51
|
return result;
|
|
52
52
|
}
|
|
53
|
+
export const defaultCrypto = {
|
|
54
|
+
h,
|
|
55
|
+
hi,
|
|
56
|
+
hmac,
|
|
57
|
+
xor
|
|
58
|
+
};
|
|
53
59
|
// Implements https://datatracker.ietf.org/doc/html/rfc5802#section-9
|
|
54
|
-
export async function authenticate(authenticateAPI, connection, algorithm, username, password) {
|
|
60
|
+
export async function authenticate(authenticateAPI, connection, algorithm, username, password, crypto = defaultCrypto) {
|
|
61
|
+
const { h, hi, hmac, xor } = crypto;
|
|
55
62
|
const definition = ScramAlgorithms[algorithm];
|
|
56
63
|
if (!definition) {
|
|
57
64
|
throw new AuthenticationError(`Unsupported SCRAM algorithm ${algorithm}`);
|
|
@@ -99,6 +106,5 @@ export async function authenticate(authenticateAPI, connection, algorithm, usern
|
|
|
99
106
|
else if (lastData.v !== serverSignature.toString('base64')) {
|
|
100
107
|
throw new AuthenticationError('Invalid server signature.');
|
|
101
108
|
}
|
|
102
|
-
/* c8 ignore next 2 */
|
|
103
109
|
return lastResponse;
|
|
104
110
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const kInstance: unique symbol;
|
package/dist/symbols.js
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platformatic/kafka",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Modern and performant client for Apache Kafka",
|
|
5
5
|
"homepage": "https://github.com/platformatic/kafka",
|
|
6
6
|
"author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
|
|
@@ -24,6 +24,20 @@
|
|
|
24
24
|
"type": "module",
|
|
25
25
|
"exports": "./dist/index.js",
|
|
26
26
|
"types": "./dist/index.d.ts",
|
|
27
|
+
"scripts": {
|
|
28
|
+
"build": "rm -rf dist && tsc -p tsconfig.base.json",
|
|
29
|
+
"lint": "eslint --cache",
|
|
30
|
+
"typecheck": "tsc -p . --noEmit",
|
|
31
|
+
"format": "prettier -w benchmarks playground src test",
|
|
32
|
+
"test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
|
|
33
|
+
"test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
|
|
34
|
+
"ci": "npm run build && npm run lint && npm run test:ci",
|
|
35
|
+
"prepublishOnly": "npm run build && npm run lint",
|
|
36
|
+
"postpublish": "git push origin && git push origin -f --tags",
|
|
37
|
+
"generate:apis": "node --experimental-strip-types scripts/generate-apis.ts",
|
|
38
|
+
"generate:errors": "node --experimental-strip-types scripts/generate-errors.ts",
|
|
39
|
+
"create:api": "node --experimental-strip-types scripts/create-api.ts"
|
|
40
|
+
},
|
|
27
41
|
"dependencies": {
|
|
28
42
|
"@watchable/unpromise": "^1.0.2",
|
|
29
43
|
"ajv": "^8.17.1",
|
|
@@ -59,17 +73,5 @@
|
|
|
59
73
|
},
|
|
60
74
|
"engines": {
|
|
61
75
|
"node": ">= 22.14.0"
|
|
62
|
-
},
|
|
63
|
-
"scripts": {
|
|
64
|
-
"build": "rm -rf dist && tsc -p tsconfig.base.json",
|
|
65
|
-
"lint": "eslint --cache",
|
|
66
|
-
"typecheck": "tsc -p . --noEmit",
|
|
67
|
-
"format": "prettier -w benchmarks playground src test",
|
|
68
|
-
"test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
|
|
69
|
-
"test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
|
|
70
|
-
"ci": "npm run build && npm run lint && npm run test:ci",
|
|
71
|
-
"generate:apis": "node --experimental-strip-types scripts/generate-apis.ts",
|
|
72
|
-
"generate:errors": "node --experimental-strip-types scripts/generate-errors.ts",
|
|
73
|
-
"create:api": "node --experimental-strip-types scripts/create-api.ts"
|
|
74
76
|
}
|
|
75
77
|
}
|