kafka-ts 1.1.6-beta.1 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cluster.js +17 -0
- package/dist/producer/producer.js +54 -54
- package/dist/utils/tracer.js +1 -1
- package/package.json +1 -1
package/dist/cluster.js
CHANGED
|
@@ -1,10 +1,21 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.Cluster = void 0;
|
|
4
13
|
const api_1 = require("./api");
|
|
5
14
|
const broker_1 = require("./broker");
|
|
6
15
|
const error_1 = require("./utils/error");
|
|
7
16
|
const logger_1 = require("./utils/logger");
|
|
17
|
+
const tracer_1 = require("./utils/tracer");
|
|
18
|
+
const trace = (0, tracer_1.createTracer)('Cluster');
|
|
8
19
|
class Cluster {
|
|
9
20
|
options;
|
|
10
21
|
seedBroker;
|
|
@@ -88,3 +99,9 @@ class Cluster {
|
|
|
88
99
|
}
|
|
89
100
|
}
|
|
90
101
|
exports.Cluster = Cluster;
|
|
102
|
+
__decorate([
|
|
103
|
+
trace((nodeId) => ({ nodeId, result: `<Broker ${nodeId}>` })),
|
|
104
|
+
__metadata("design:type", Function),
|
|
105
|
+
__metadata("design:paramtypes", [Number]),
|
|
106
|
+
__metadata("design:returntype", Promise)
|
|
107
|
+
], Cluster.prototype, "acquireBroker", null);
|
|
@@ -53,70 +53,70 @@ class Producer {
|
|
|
53
53
|
const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
|
|
54
54
|
try {
|
|
55
55
|
await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
baseTimestamp
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
maxTimestamp
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
lastOffsetDelta: messages.length - 1,
|
|
77
|
-
baseTimestamp: baseTimestamp ?? 0n,
|
|
78
|
-
maxTimestamp: maxTimestamp ?? 0n,
|
|
79
|
-
producerId: this.producerId,
|
|
80
|
-
producerEpoch: 0,
|
|
81
|
-
baseSequence: this.getSequence(topic, partitionIndex),
|
|
82
|
-
records: messages.map((message, index) => ({
|
|
56
|
+
try {
|
|
57
|
+
await this.lock.acquire([`node:${nodeId}`], async () => {
|
|
58
|
+
const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
|
|
59
|
+
name: topic,
|
|
60
|
+
partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
|
|
61
|
+
const partitionIndex = parseInt(partition);
|
|
62
|
+
let baseTimestamp;
|
|
63
|
+
let maxTimestamp;
|
|
64
|
+
messages.forEach(({ timestamp = defaultTimestamp }) => {
|
|
65
|
+
if (!baseTimestamp || timestamp < baseTimestamp) {
|
|
66
|
+
baseTimestamp = timestamp;
|
|
67
|
+
}
|
|
68
|
+
if (!maxTimestamp || timestamp > maxTimestamp) {
|
|
69
|
+
maxTimestamp = timestamp;
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
return {
|
|
73
|
+
index: partitionIndex,
|
|
74
|
+
baseOffset: 0n,
|
|
75
|
+
partitionLeaderEpoch: -1,
|
|
83
76
|
attributes: 0,
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
77
|
+
lastOffsetDelta: messages.length - 1,
|
|
78
|
+
baseTimestamp: baseTimestamp ?? 0n,
|
|
79
|
+
maxTimestamp: maxTimestamp ?? 0n,
|
|
80
|
+
producerId: this.producerId,
|
|
81
|
+
producerEpoch: 0,
|
|
82
|
+
baseSequence: this.getSequence(topic, partitionIndex),
|
|
83
|
+
records: messages.map((message, index) => ({
|
|
84
|
+
attributes: 0,
|
|
85
|
+
timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
|
|
86
|
+
offsetDelta: index,
|
|
87
|
+
key: message.key ?? null,
|
|
88
|
+
value: message.value,
|
|
89
|
+
headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
|
|
90
|
+
key,
|
|
91
|
+
value,
|
|
92
|
+
})),
|
|
91
93
|
})),
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
})
|
|
95
|
-
}));
|
|
96
|
-
try {
|
|
94
|
+
};
|
|
95
|
+
}),
|
|
96
|
+
}));
|
|
97
97
|
await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
|
|
98
98
|
transactionalId: null,
|
|
99
99
|
acks,
|
|
100
100
|
timeoutMs: 30000,
|
|
101
101
|
topicData,
|
|
102
102
|
});
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
await this.cluster.refreshBrokerMetadata();
|
|
108
|
-
await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
|
|
109
|
-
const messages = Object.values(topicPartitionMessages).flatMap(partitionMessages => Object.values(partitionMessages).flat()).map(({ partition, ...message }) => message);
|
|
110
|
-
return this.send(messages, { acks });
|
|
111
|
-
}
|
|
112
|
-
throw error;
|
|
113
|
-
}
|
|
114
|
-
topicData.forEach(({ name, partitionData }) => {
|
|
115
|
-
partitionData.forEach(({ index, records }) => {
|
|
116
|
-
this.updateSequence(name, index, records.length);
|
|
103
|
+
topicData.forEach(({ name, partitionData }) => {
|
|
104
|
+
partitionData.forEach(({ index, records }) => {
|
|
105
|
+
this.updateSequence(name, index, records.length);
|
|
106
|
+
});
|
|
117
107
|
});
|
|
118
108
|
});
|
|
119
|
-
}
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
if (error instanceof error_1.BrokerNotAvailableError || (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER)) {
|
|
112
|
+
logger_1.log.debug('Refreshing broker metadata', { reason: error.message, nodeId });
|
|
113
|
+
await this.cluster.refreshBrokerMetadata();
|
|
114
|
+
await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
|
|
115
|
+
const messages = Object.values(topicPartitionMessages).flatMap(partitionMessages => Object.values(partitionMessages).flat()).map(({ partition, ...message }) => message);
|
|
116
|
+
return this.send(messages, { acks });
|
|
117
|
+
}
|
|
118
|
+
throw error;
|
|
119
|
+
}
|
|
120
120
|
}));
|
|
121
121
|
}
|
|
122
122
|
catch (error) {
|
package/dist/utils/tracer.js
CHANGED
|
@@ -11,8 +11,8 @@ class DebugTracer {
|
|
|
11
11
|
const startTime = Date.now();
|
|
12
12
|
const onEnd = (result) => {
|
|
13
13
|
logger_1.log.debug(`[${module}.${method}] ${metadata?.message ?? ''} +${Date.now() - startTime}ms`, {
|
|
14
|
-
...metadata,
|
|
15
14
|
...(!!result && { result }),
|
|
15
|
+
...metadata,
|
|
16
16
|
});
|
|
17
17
|
return result;
|
|
18
18
|
};
|