kafka-ts 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/producer/producer.d.ts +1 -0
- package/dist/producer/producer.js +51 -46
- package/dist/utils/lock.d.ts +9 -0
- package/dist/utils/lock.js +44 -0
- package/package.json +1 -1
|
@@ -16,6 +16,7 @@ const partitioner_1 = require("../distributors/partitioner");
|
|
|
16
16
|
const metadata_1 = require("../metadata");
|
|
17
17
|
const delay_1 = require("../utils/delay");
|
|
18
18
|
const error_1 = require("../utils/error");
|
|
19
|
+
const lock_1 = require("../utils/lock");
|
|
19
20
|
const tracer_1 = require("../utils/tracer");
|
|
20
21
|
const trace = (0, tracer_1.createTracer)('Producer');
|
|
21
22
|
class Producer {
|
|
@@ -26,6 +27,7 @@ class Producer {
|
|
|
26
27
|
producerEpoch = 0;
|
|
27
28
|
sequences = {};
|
|
28
29
|
partition;
|
|
30
|
+
lock = new lock_1.Lock();
|
|
29
31
|
constructor(cluster, options) {
|
|
30
32
|
this.cluster = cluster;
|
|
31
33
|
this.options = {
|
|
@@ -49,54 +51,57 @@ class Producer {
|
|
|
49
51
|
const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
|
|
50
52
|
try {
|
|
51
53
|
await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
baseTimestamp: baseTimestamp ?? 0n,
|
|
73
|
-
maxTimestamp: maxTimestamp ?? 0n,
|
|
74
|
-
producerId: this.producerId,
|
|
75
|
-
producerEpoch: 0,
|
|
76
|
-
baseSequence: this.getSequence(topic, partitionIndex),
|
|
77
|
-
records: messages.map((message, index) => ({
|
|
54
|
+
const lockKeys = Object.entries(topicPartitionMessages).flatMap(([topic, partitionMessages]) => Object.entries(partitionMessages).map(([partition]) => `${topic}-${partition}`));
|
|
55
|
+
await this.lock.acquire(lockKeys, async () => {
|
|
56
|
+
const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
|
|
57
|
+
name: topic,
|
|
58
|
+
partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
|
|
59
|
+
const partitionIndex = parseInt(partition);
|
|
60
|
+
let baseTimestamp;
|
|
61
|
+
let maxTimestamp;
|
|
62
|
+
messages.forEach(({ timestamp = defaultTimestamp }) => {
|
|
63
|
+
if (!baseTimestamp || timestamp < baseTimestamp) {
|
|
64
|
+
baseTimestamp = timestamp;
|
|
65
|
+
}
|
|
66
|
+
if (!maxTimestamp || timestamp > maxTimestamp) {
|
|
67
|
+
maxTimestamp = timestamp;
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
return {
|
|
71
|
+
index: partitionIndex,
|
|
72
|
+
baseOffset: 0n,
|
|
73
|
+
partitionLeaderEpoch: -1,
|
|
78
74
|
attributes: 0,
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
75
|
+
lastOffsetDelta: messages.length - 1,
|
|
76
|
+
baseTimestamp: baseTimestamp ?? 0n,
|
|
77
|
+
maxTimestamp: maxTimestamp ?? 0n,
|
|
78
|
+
producerId: this.producerId,
|
|
79
|
+
producerEpoch: 0,
|
|
80
|
+
baseSequence: this.getSequence(topic, partitionIndex),
|
|
81
|
+
records: messages.map((message, index) => ({
|
|
82
|
+
attributes: 0,
|
|
83
|
+
timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
|
|
84
|
+
offsetDelta: index,
|
|
85
|
+
key: message.key ?? null,
|
|
86
|
+
value: message.value,
|
|
87
|
+
headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
|
|
88
|
+
key,
|
|
89
|
+
value,
|
|
90
|
+
})),
|
|
86
91
|
})),
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
})
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
92
|
+
};
|
|
93
|
+
}),
|
|
94
|
+
}));
|
|
95
|
+
await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
|
|
96
|
+
transactionalId: null,
|
|
97
|
+
acks,
|
|
98
|
+
timeoutMs: 5000,
|
|
99
|
+
topicData,
|
|
100
|
+
});
|
|
101
|
+
topicData.forEach(({ name, partitionData }) => {
|
|
102
|
+
partitionData.forEach(({ index, records }) => {
|
|
103
|
+
this.updateSequence(name, index, records.length);
|
|
104
|
+
});
|
|
100
105
|
});
|
|
101
106
|
});
|
|
102
107
|
}));
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import EventEmitter from 'events';
|
|
3
|
+
export declare class Lock extends EventEmitter {
|
|
4
|
+
private locks;
|
|
5
|
+
constructor();
|
|
6
|
+
acquire(keys: string[], callback: () => Promise<void>): Promise<void>;
|
|
7
|
+
private acquireKey;
|
|
8
|
+
private releaseKey;
|
|
9
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Lock = void 0;
|
|
7
|
+
const events_1 = __importDefault(require("events"));
|
|
8
|
+
const logger_1 = require("./logger");
|
|
9
|
+
class Lock extends events_1.default {
|
|
10
|
+
locks = {};
|
|
11
|
+
constructor() {
|
|
12
|
+
super();
|
|
13
|
+
this.setMaxListeners(Infinity);
|
|
14
|
+
}
|
|
15
|
+
async acquire(keys, callback) {
|
|
16
|
+
await Promise.all(keys.map((key) => this.acquireKey(key)));
|
|
17
|
+
try {
|
|
18
|
+
await callback();
|
|
19
|
+
}
|
|
20
|
+
finally {
|
|
21
|
+
keys.forEach((key) => this.releaseKey(key));
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
async acquireKey(key) {
|
|
25
|
+
while (this.locks[key]) {
|
|
26
|
+
await new Promise((resolve) => {
|
|
27
|
+
const timeout = setTimeout(() => {
|
|
28
|
+
logger_1.log.warn(`Lock timed out`, { key });
|
|
29
|
+
this.emit(`release:${key}`);
|
|
30
|
+
}, 10_000);
|
|
31
|
+
this.once(`release:${key}`, () => {
|
|
32
|
+
clearTimeout(timeout);
|
|
33
|
+
resolve();
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
this.locks[key] = true;
|
|
38
|
+
}
|
|
39
|
+
releaseKey(key) {
|
|
40
|
+
this.locks[key] = false;
|
|
41
|
+
this.emit(`release:${key}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
exports.Lock = Lock;
|