kafka-ts 1.1.5 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/create-topics.d.ts +2 -2
- package/dist/api/delete-topics.d.ts +1 -1
- package/dist/api/index.d.ts +12 -13
- package/dist/api/join-group.d.ts +1 -2
- package/dist/api/metadata.d.ts +3 -3
- package/dist/api/sasl-authenticate.d.ts +1 -2
- package/dist/auth/scram.js +1 -1
- package/dist/broker.d.ts +0 -2
- package/dist/client.d.ts +0 -2
- package/dist/cluster.d.ts +1 -2
- package/dist/cluster.js +21 -1
- package/dist/cluster.test.js +1 -1
- package/dist/codecs/types.d.ts +0 -1
- package/dist/connection.d.ts +0 -2
- package/dist/connection.js +17 -7
- package/dist/consumer/consumer.d.ts +0 -1
- package/dist/consumer/consumer.js +31 -21
- package/dist/consumer/fetcher.d.ts +0 -2
- package/dist/consumer/processor.d.ts +0 -2
- package/dist/producer/producer.js +60 -48
- package/dist/utils/crypto.d.ts +5 -6
- package/dist/utils/decoder.d.ts +3 -4
- package/dist/utils/encoder.d.ts +2 -3
- package/dist/utils/error.d.ts +4 -0
- package/dist/utils/error.js +9 -1
- package/dist/utils/lock.d.ts +0 -1
- package/dist/utils/murmur2.d.ts +0 -1
- package/dist/utils/tracer.js +1 -1
- package/package.json +7 -7
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
**KafkaTS** is a Apache Kafka client library for Node.js. It provides both a low-level API for communicating directly with the Apache Kafka cluster and high-level APIs for publishing and subscribing to Kafka topics.
|
|
4
4
|
|
|
5
|
-
**Supported Kafka versions:** 3.6
|
|
5
|
+
**Supported Kafka versions:** ^3.6.x, ^4.0.0
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
@@ -12,8 +12,8 @@ export declare const CREATE_TOPICS: import("../utils/api").Api<{
|
|
|
12
12
|
value: string | null;
|
|
13
13
|
}[];
|
|
14
14
|
}[];
|
|
15
|
-
timeoutMs?: number
|
|
16
|
-
validateOnly?: boolean
|
|
15
|
+
timeoutMs?: number;
|
|
16
|
+
validateOnly?: boolean;
|
|
17
17
|
}, {
|
|
18
18
|
_tag: void;
|
|
19
19
|
throttleTimeMs: number;
|
package/dist/api/index.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
import { Api } from '../utils/api';
|
|
3
2
|
export declare const API: {
|
|
4
3
|
API_VERSIONS: Api<unknown, {
|
|
@@ -13,19 +12,19 @@ export declare const API: {
|
|
|
13
12
|
CREATE_TOPICS: Api<{
|
|
14
13
|
topics: {
|
|
15
14
|
name: string;
|
|
16
|
-
numPartitions?: number
|
|
17
|
-
replicationFactor?: number
|
|
15
|
+
numPartitions?: number;
|
|
16
|
+
replicationFactor?: number;
|
|
18
17
|
assignments?: {
|
|
19
18
|
partitionIndex: number;
|
|
20
19
|
brokerIds: number[];
|
|
21
|
-
}[]
|
|
20
|
+
}[];
|
|
22
21
|
configs?: {
|
|
23
22
|
name: string;
|
|
24
23
|
value: string | null;
|
|
25
|
-
}[]
|
|
24
|
+
}[];
|
|
26
25
|
}[];
|
|
27
|
-
timeoutMs?: number
|
|
28
|
-
validateOnly?: boolean
|
|
26
|
+
timeoutMs?: number;
|
|
27
|
+
validateOnly?: boolean;
|
|
29
28
|
}, {
|
|
30
29
|
_tag: void;
|
|
31
30
|
throttleTimeMs: number;
|
|
@@ -53,7 +52,7 @@ export declare const API: {
|
|
|
53
52
|
name: string | null;
|
|
54
53
|
topicId: string | null;
|
|
55
54
|
}[];
|
|
56
|
-
timeoutMs?: number
|
|
55
|
+
timeoutMs?: number;
|
|
57
56
|
}, {
|
|
58
57
|
_tag: void;
|
|
59
58
|
throttleTimeMs: number;
|
|
@@ -213,7 +212,7 @@ export declare const API: {
|
|
|
213
212
|
members: {
|
|
214
213
|
memberId: string;
|
|
215
214
|
groupInstanceId: string | null;
|
|
216
|
-
metadata: Buffer
|
|
215
|
+
metadata: Buffer<ArrayBufferLike>;
|
|
217
216
|
_tag: void;
|
|
218
217
|
}[];
|
|
219
218
|
_tag2: void;
|
|
@@ -269,9 +268,9 @@ export declare const API: {
|
|
|
269
268
|
topics?: {
|
|
270
269
|
id: string | null;
|
|
271
270
|
name: string;
|
|
272
|
-
}[] | null
|
|
273
|
-
allowTopicAutoCreation?: boolean
|
|
274
|
-
includeTopicAuthorizedOperations?: boolean
|
|
271
|
+
}[] | null;
|
|
272
|
+
allowTopicAutoCreation?: boolean;
|
|
273
|
+
includeTopicAuthorizedOperations?: boolean;
|
|
275
274
|
}, {
|
|
276
275
|
_tag: void;
|
|
277
276
|
throttleTimeMs: number;
|
|
@@ -422,7 +421,7 @@ export declare const API: {
|
|
|
422
421
|
_tag: void;
|
|
423
422
|
errorCode: number;
|
|
424
423
|
errorMessage: string | null;
|
|
425
|
-
authBytes: Buffer | null;
|
|
424
|
+
authBytes: Buffer<ArrayBufferLike> | null;
|
|
426
425
|
sessionLifetimeMs: bigint;
|
|
427
426
|
_tag2: void;
|
|
428
427
|
}>;
|
package/dist/api/join-group.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
export declare const JOIN_GROUP: import("../utils/api").Api<{
|
|
3
2
|
groupId: string;
|
|
4
3
|
sessionTimeoutMs: number;
|
|
@@ -27,7 +26,7 @@ export declare const JOIN_GROUP: import("../utils/api").Api<{
|
|
|
27
26
|
members: {
|
|
28
27
|
memberId: string;
|
|
29
28
|
groupInstanceId: string | null;
|
|
30
|
-
metadata: Buffer
|
|
29
|
+
metadata: Buffer<ArrayBufferLike>;
|
|
31
30
|
_tag: void;
|
|
32
31
|
}[];
|
|
33
32
|
_tag2: void;
|
package/dist/api/metadata.d.ts
CHANGED
|
@@ -3,9 +3,9 @@ export declare const METADATA: import("../utils/api").Api<{
|
|
|
3
3
|
topics?: {
|
|
4
4
|
id: string | null;
|
|
5
5
|
name: string;
|
|
6
|
-
}[] | null
|
|
7
|
-
allowTopicAutoCreation?: boolean
|
|
8
|
-
includeTopicAuthorizedOperations?: boolean
|
|
6
|
+
}[] | null;
|
|
7
|
+
allowTopicAutoCreation?: boolean;
|
|
8
|
+
includeTopicAuthorizedOperations?: boolean;
|
|
9
9
|
}, {
|
|
10
10
|
_tag: void;
|
|
11
11
|
throttleTimeMs: number;
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
export declare const SASL_AUTHENTICATE: import("../utils/api").Api<{
|
|
3
2
|
authBytes: Buffer;
|
|
4
3
|
}, {
|
|
5
4
|
_tag: void;
|
|
6
5
|
errorCode: number;
|
|
7
6
|
errorMessage: string | null;
|
|
8
|
-
authBytes: Buffer | null;
|
|
7
|
+
authBytes: Buffer<ArrayBufferLike> | null;
|
|
9
8
|
sessionLifetimeMs: bigint;
|
|
10
9
|
_tag2: void;
|
|
11
10
|
}>;
|
package/dist/auth/scram.js
CHANGED
|
@@ -28,7 +28,7 @@ const saslScram = ({ mechanism, keyLength, digest }) => ({ username, password })
|
|
|
28
28
|
const saltedPassword = await (0, crypto_1.saltPassword)(password, salt, iterations, keyLength, digest);
|
|
29
29
|
const clientKey = (0, crypto_1.hmac)(saltedPassword, 'Client Key', digest);
|
|
30
30
|
const clientKeyHash = (0, crypto_1.hash)(clientKey, digest);
|
|
31
|
-
let finalMessage = `c=${(0, crypto_1.base64Encode)('n,,')},r=${rnonce}`;
|
|
31
|
+
let finalMessage = `c=${(0, crypto_1.base64Encode)(Buffer.from('n,,'))},r=${rnonce}`;
|
|
32
32
|
const fullMessage = `${firstMessage},${authBytes.toString()},${finalMessage}`;
|
|
33
33
|
const clientSignature = (0, crypto_1.hmac)(clientKeyHash, fullMessage, digest);
|
|
34
34
|
const clientProof = (0, crypto_1.base64Encode)((0, crypto_1.xor)(clientKey, clientSignature));
|
package/dist/broker.d.ts
CHANGED
package/dist/client.d.ts
CHANGED
package/dist/cluster.d.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
/// <reference types="node" />
|
|
3
1
|
import { TcpSocketConnectOpts } from 'net';
|
|
4
2
|
import { TLSSocketOptions } from 'tls';
|
|
5
3
|
import { Broker, SASLProvider } from './broker';
|
|
@@ -18,6 +16,7 @@ export declare class Cluster {
|
|
|
18
16
|
private brokerMetadata;
|
|
19
17
|
constructor(options: ClusterOptions);
|
|
20
18
|
connect(): Promise<void>;
|
|
19
|
+
refreshBrokerMetadata(): Promise<void>;
|
|
21
20
|
ensureConnected(): Promise<void>;
|
|
22
21
|
disconnect(): Promise<void>;
|
|
23
22
|
setSeedBroker: (nodeId: number) => Promise<void>;
|
package/dist/cluster.js
CHANGED
|
@@ -1,10 +1,21 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.Cluster = void 0;
|
|
4
13
|
const api_1 = require("./api");
|
|
5
14
|
const broker_1 = require("./broker");
|
|
6
15
|
const error_1 = require("./utils/error");
|
|
7
16
|
const logger_1 = require("./utils/logger");
|
|
17
|
+
const tracer_1 = require("./utils/tracer");
|
|
18
|
+
const trace = (0, tracer_1.createTracer)('Cluster');
|
|
8
19
|
class Cluster {
|
|
9
20
|
options;
|
|
10
21
|
seedBroker;
|
|
@@ -16,6 +27,9 @@ class Cluster {
|
|
|
16
27
|
async connect() {
|
|
17
28
|
this.seedBroker = await this.findSeedBroker();
|
|
18
29
|
this.brokerById = {};
|
|
30
|
+
await this.refreshBrokerMetadata();
|
|
31
|
+
}
|
|
32
|
+
async refreshBrokerMetadata() {
|
|
19
33
|
const metadata = await this.sendRequest(api_1.API.METADATA, { topics: [] });
|
|
20
34
|
this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
|
|
21
35
|
}
|
|
@@ -51,7 +65,7 @@ class Cluster {
|
|
|
51
65
|
};
|
|
52
66
|
async acquireBroker(nodeId) {
|
|
53
67
|
if (!(nodeId in this.brokerMetadata)) {
|
|
54
|
-
throw new error_1.
|
|
68
|
+
throw new error_1.BrokerNotAvailableError(nodeId);
|
|
55
69
|
}
|
|
56
70
|
const broker = new broker_1.Broker({
|
|
57
71
|
clientId: this.options.clientId,
|
|
@@ -85,3 +99,9 @@ class Cluster {
|
|
|
85
99
|
}
|
|
86
100
|
}
|
|
87
101
|
exports.Cluster = Cluster;
|
|
102
|
+
__decorate([
|
|
103
|
+
trace((nodeId) => ({ nodeId, result: `<Broker ${nodeId}>` })),
|
|
104
|
+
__metadata("design:type", Function),
|
|
105
|
+
__metadata("design:paramtypes", [Number]),
|
|
106
|
+
__metadata("design:returntype", Promise)
|
|
107
|
+
], Cluster.prototype, "acquireBroker", null);
|
package/dist/cluster.test.js
CHANGED
|
@@ -9,7 +9,7 @@ const auth_1 = require("./auth");
|
|
|
9
9
|
const client_1 = require("./client");
|
|
10
10
|
const kafka = (0, client_1.createKafkaClient)({
|
|
11
11
|
clientId: 'kafka-ts',
|
|
12
|
-
bootstrapServers: [{ host: 'localhost', port:
|
|
12
|
+
bootstrapServers: [{ host: 'localhost', port: 39092 }],
|
|
13
13
|
sasl: (0, auth_1.saslPlain)({ username: 'admin', password: 'admin' }),
|
|
14
14
|
ssl: { ca: (0, fs_1.readFileSync)('./certs/ca.crt').toString() },
|
|
15
15
|
});
|
package/dist/codecs/types.d.ts
CHANGED
package/dist/connection.d.ts
CHANGED
package/dist/connection.js
CHANGED
|
@@ -21,13 +21,23 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
21
21
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
22
22
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
23
23
|
};
|
|
24
|
-
var __importStar = (this && this.__importStar) || function (
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
};
|
|
24
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
25
|
+
var ownKeys = function(o) {
|
|
26
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
27
|
+
var ar = [];
|
|
28
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
29
|
+
return ar;
|
|
30
|
+
};
|
|
31
|
+
return ownKeys(o);
|
|
32
|
+
};
|
|
33
|
+
return function (mod) {
|
|
34
|
+
if (mod && mod.__esModule) return mod;
|
|
35
|
+
var result = {};
|
|
36
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
37
|
+
__setModuleDefault(result, mod);
|
|
38
|
+
return result;
|
|
39
|
+
};
|
|
40
|
+
})();
|
|
31
41
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
32
42
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
33
43
|
};
|
|
@@ -206,30 +206,40 @@ class Consumer extends events_1.default {
|
|
|
206
206
|
await this.consumerGroup?.offsetCommit(topicPartitions);
|
|
207
207
|
this.offsetManager.flush(topicPartitions);
|
|
208
208
|
}
|
|
209
|
-
fetch(nodeId, assignment) {
|
|
209
|
+
async fetch(nodeId, assignment) {
|
|
210
210
|
const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;
|
|
211
211
|
this.consumerGroup?.handleLastHeartbeat();
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
partition
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
212
|
+
try {
|
|
213
|
+
return await this.cluster.sendRequestToNode(nodeId)(api_1.API.FETCH, {
|
|
214
|
+
maxWaitMs,
|
|
215
|
+
minBytes,
|
|
216
|
+
maxBytes,
|
|
217
|
+
isolationLevel,
|
|
218
|
+
sessionId: 0,
|
|
219
|
+
sessionEpoch: -1,
|
|
220
|
+
topics: Object.entries(assignment).map(([topic, partitions]) => ({
|
|
221
|
+
topicId: this.metadata.getTopicIdByName(topic),
|
|
222
|
+
partitions: partitions.map((partition) => ({
|
|
223
|
+
partition,
|
|
224
|
+
currentLeaderEpoch: -1,
|
|
225
|
+
fetchOffset: this.offsetManager.getCurrentOffset(topic, partition),
|
|
226
|
+
lastFetchedEpoch: -1,
|
|
227
|
+
logStartOffset: -1n,
|
|
228
|
+
partitionMaxBytes,
|
|
229
|
+
})),
|
|
228
230
|
})),
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
}
|
|
231
|
+
forgottenTopicsData: [],
|
|
232
|
+
rackId,
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
catch (error) {
|
|
236
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OFFSET_OUT_OF_RANGE) {
|
|
237
|
+
logger_1.log.warn('Offset out of range. Resetting offsets.');
|
|
238
|
+
await this.offsetManager.fetchOffsets({ fromTimestamp: this.options.fromTimestamp });
|
|
239
|
+
return this.fetch(nodeId, assignment);
|
|
240
|
+
}
|
|
241
|
+
throw error;
|
|
242
|
+
}
|
|
233
243
|
}
|
|
234
244
|
}
|
|
235
245
|
exports.Consumer = Consumer;
|
|
@@ -53,58 +53,70 @@ class Producer {
|
|
|
53
53
|
const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
|
|
54
54
|
try {
|
|
55
55
|
await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
baseTimestamp
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
maxTimestamp
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
lastOffsetDelta: messages.length - 1,
|
|
77
|
-
baseTimestamp: baseTimestamp ?? 0n,
|
|
78
|
-
maxTimestamp: maxTimestamp ?? 0n,
|
|
79
|
-
producerId: this.producerId,
|
|
80
|
-
producerEpoch: 0,
|
|
81
|
-
baseSequence: this.getSequence(topic, partitionIndex),
|
|
82
|
-
records: messages.map((message, index) => ({
|
|
56
|
+
try {
|
|
57
|
+
await this.lock.acquire([`node:${nodeId}`], async () => {
|
|
58
|
+
const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
|
|
59
|
+
name: topic,
|
|
60
|
+
partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
|
|
61
|
+
const partitionIndex = parseInt(partition);
|
|
62
|
+
let baseTimestamp;
|
|
63
|
+
let maxTimestamp;
|
|
64
|
+
messages.forEach(({ timestamp = defaultTimestamp }) => {
|
|
65
|
+
if (!baseTimestamp || timestamp < baseTimestamp) {
|
|
66
|
+
baseTimestamp = timestamp;
|
|
67
|
+
}
|
|
68
|
+
if (!maxTimestamp || timestamp > maxTimestamp) {
|
|
69
|
+
maxTimestamp = timestamp;
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
return {
|
|
73
|
+
index: partitionIndex,
|
|
74
|
+
baseOffset: 0n,
|
|
75
|
+
partitionLeaderEpoch: -1,
|
|
83
76
|
attributes: 0,
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
77
|
+
lastOffsetDelta: messages.length - 1,
|
|
78
|
+
baseTimestamp: baseTimestamp ?? 0n,
|
|
79
|
+
maxTimestamp: maxTimestamp ?? 0n,
|
|
80
|
+
producerId: this.producerId,
|
|
81
|
+
producerEpoch: 0,
|
|
82
|
+
baseSequence: this.getSequence(topic, partitionIndex),
|
|
83
|
+
records: messages.map((message, index) => ({
|
|
84
|
+
attributes: 0,
|
|
85
|
+
timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
|
|
86
|
+
offsetDelta: index,
|
|
87
|
+
key: message.key ?? null,
|
|
88
|
+
value: message.value,
|
|
89
|
+
headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
|
|
90
|
+
key,
|
|
91
|
+
value,
|
|
92
|
+
})),
|
|
91
93
|
})),
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
})
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
94
|
+
};
|
|
95
|
+
}),
|
|
96
|
+
}));
|
|
97
|
+
await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
|
|
98
|
+
transactionalId: null,
|
|
99
|
+
acks,
|
|
100
|
+
timeoutMs: 30000,
|
|
101
|
+
topicData,
|
|
102
|
+
});
|
|
103
|
+
topicData.forEach(({ name, partitionData }) => {
|
|
104
|
+
partitionData.forEach(({ index, records }) => {
|
|
105
|
+
this.updateSequence(name, index, records.length);
|
|
106
|
+
});
|
|
105
107
|
});
|
|
106
108
|
});
|
|
107
|
-
}
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
if (error instanceof error_1.BrokerNotAvailableError || (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER)) {
|
|
112
|
+
logger_1.log.debug('Refreshing broker metadata', { reason: error.message, nodeId });
|
|
113
|
+
await this.cluster.refreshBrokerMetadata();
|
|
114
|
+
await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
|
|
115
|
+
const messages = Object.values(topicPartitionMessages).flatMap(partitionMessages => Object.values(partitionMessages).flat()).map(({ partition, ...message }) => message);
|
|
116
|
+
return this.send(messages, { acks });
|
|
117
|
+
}
|
|
118
|
+
throw error;
|
|
119
|
+
}
|
|
108
120
|
}));
|
|
109
121
|
}
|
|
110
122
|
catch (error) {
|
package/dist/utils/crypto.d.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
export declare const generateNonce: () => string;
|
|
3
|
-
export declare const saltPassword: (password: string, salt: string, iterations: number, keyLength: number, digest: string) => Promise<Buffer
|
|
4
|
-
export declare const base64Encode: (input: Buffer
|
|
2
|
+
export declare const saltPassword: (password: string, salt: string, iterations: number, keyLength: number, digest: string) => Promise<Buffer<ArrayBufferLike>>;
|
|
3
|
+
export declare const base64Encode: (input: Buffer) => string;
|
|
5
4
|
export declare const base64Decode: (input: string) => string;
|
|
6
|
-
export declare const hash: (data: Buffer, digest: string) => Buffer
|
|
7
|
-
export declare const hmac: (key: Buffer, data: Buffer | string, digest: string) => Buffer
|
|
8
|
-
export declare const xor: (a: Buffer, b: Buffer) => Buffer
|
|
5
|
+
export declare const hash: (data: Buffer, digest: string) => Buffer<ArrayBufferLike>;
|
|
6
|
+
export declare const hmac: (key: Buffer, data: Buffer | string, digest: string) => Buffer<ArrayBufferLike>;
|
|
7
|
+
export declare const xor: (a: Buffer, b: Buffer) => Buffer<ArrayBuffer>;
|
package/dist/utils/decoder.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
export declare class Decoder {
|
|
3
2
|
private buffer;
|
|
4
3
|
private offset;
|
|
@@ -24,8 +23,8 @@ export declare class Decoder {
|
|
|
24
23
|
readCompactArray<T>(callback: (opts: Decoder) => T): T[];
|
|
25
24
|
readVarIntArray<T>(callback: (opts: Decoder) => T): T[];
|
|
26
25
|
readRecords<T>(callback: (opts: Decoder) => T): T[];
|
|
27
|
-
read(length?: number): Buffer
|
|
28
|
-
readBytes(): Buffer
|
|
29
|
-
readCompactBytes(): Buffer | null;
|
|
26
|
+
read(length?: number): Buffer<ArrayBufferLike>;
|
|
27
|
+
readBytes(): Buffer<ArrayBufferLike>;
|
|
28
|
+
readCompactBytes(): Buffer<ArrayBufferLike> | null;
|
|
30
29
|
readTagBuffer(): void;
|
|
31
30
|
}
|
package/dist/utils/encoder.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
export declare class Encoder {
|
|
3
2
|
private chunks;
|
|
4
|
-
getChunks(): Buffer[];
|
|
3
|
+
getChunks(): Buffer<ArrayBufferLike>[];
|
|
5
4
|
getBufferLength(): number;
|
|
6
5
|
write(...buffers: Buffer[]): this;
|
|
7
6
|
writeEncoder(encoder: Encoder): this;
|
|
@@ -24,5 +23,5 @@ export declare class Encoder {
|
|
|
24
23
|
writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
|
|
25
24
|
writeBytes(value: Buffer): this;
|
|
26
25
|
writeCompactBytes(value: Buffer): this;
|
|
27
|
-
value(): Buffer
|
|
26
|
+
value(): Buffer<ArrayBuffer>;
|
|
28
27
|
}
|
package/dist/utils/error.d.ts
CHANGED
|
@@ -9,5 +9,9 @@ export declare class KafkaTSApiError<T = any> extends KafkaTSError {
|
|
|
9
9
|
request: unknown | undefined;
|
|
10
10
|
constructor(errorCode: number, errorMessage: string | null, response: T);
|
|
11
11
|
}
|
|
12
|
+
export declare class BrokerNotAvailableError extends KafkaTSError {
|
|
13
|
+
brokerId: number;
|
|
14
|
+
constructor(brokerId: number);
|
|
15
|
+
}
|
|
12
16
|
export declare class ConnectionError extends KafkaTSError {
|
|
13
17
|
}
|
package/dist/utils/error.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ConnectionError = exports.KafkaTSApiError = exports.KafkaTSError = void 0;
|
|
3
|
+
exports.ConnectionError = exports.BrokerNotAvailableError = exports.KafkaTSApiError = exports.KafkaTSError = void 0;
|
|
4
4
|
const api_1 = require("../api");
|
|
5
5
|
class KafkaTSError extends Error {
|
|
6
6
|
constructor(message) {
|
|
@@ -24,6 +24,14 @@ class KafkaTSApiError extends KafkaTSError {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
exports.KafkaTSApiError = KafkaTSApiError;
|
|
27
|
+
class BrokerNotAvailableError extends KafkaTSError {
|
|
28
|
+
brokerId;
|
|
29
|
+
constructor(brokerId) {
|
|
30
|
+
super(`Broker ${brokerId} is not available`);
|
|
31
|
+
this.brokerId = brokerId;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
exports.BrokerNotAvailableError = BrokerNotAvailableError;
|
|
27
35
|
class ConnectionError extends KafkaTSError {
|
|
28
36
|
}
|
|
29
37
|
exports.ConnectionError = ConnectionError;
|
package/dist/utils/lock.d.ts
CHANGED
package/dist/utils/murmur2.d.ts
CHANGED
package/dist/utils/tracer.js
CHANGED
|
@@ -11,8 +11,8 @@ class DebugTracer {
|
|
|
11
11
|
const startTime = Date.now();
|
|
12
12
|
const onEnd = (result) => {
|
|
13
13
|
logger_1.log.debug(`[${module}.${method}] ${metadata?.message ?? ''} +${Date.now() - startTime}ms`, {
|
|
14
|
-
...metadata,
|
|
15
14
|
...(!!result && { result }),
|
|
15
|
+
...metadata,
|
|
16
16
|
});
|
|
17
17
|
return result;
|
|
18
18
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "kafka-ts",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.6",
|
|
4
4
|
"main": "dist/index.js",
|
|
5
5
|
"author": "Priit Käärd",
|
|
6
6
|
"license": "MIT",
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
"url": "https://github.com/priitkaard/kafka-ts.git"
|
|
10
10
|
},
|
|
11
11
|
"scripts": {
|
|
12
|
-
"up": "npm run down && KAFKA_VERSION=
|
|
13
|
-
"down": "KAFKA_VERSION=
|
|
12
|
+
"up": "npm run down && KAFKA_VERSION=4.0.0 docker-compose up -d && sleep 5 && bash ./scripts/create-scram-user.sh",
|
|
13
|
+
"down": "KAFKA_VERSION=4.0.0 docker-compose down",
|
|
14
14
|
"version:prerelease": "npm version prerelease --preid=beta",
|
|
15
15
|
"version:patch": "npm version patch",
|
|
16
16
|
"version:major": "npm version major",
|
|
@@ -20,10 +20,10 @@
|
|
|
20
20
|
"test": "vitest --testTimeout 60000 --bail 1"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|
|
23
|
-
"@types/node": "^
|
|
24
|
-
"prettier": "^3.
|
|
25
|
-
"typescript": "^5.
|
|
26
|
-
"vitest": "^1.
|
|
23
|
+
"@types/node": "^22.14.1",
|
|
24
|
+
"prettier": "^3.5.3",
|
|
25
|
+
"typescript": "^5.8.3",
|
|
26
|
+
"vitest": "^3.1.1"
|
|
27
27
|
},
|
|
28
28
|
"keywords": [
|
|
29
29
|
"kafka",
|