kafka-ts 0.0.1-beta.3 → 0.0.1-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/examples/src/consumer.ts +7 -1
- package/examples/src/create-topic.ts +3 -3
- package/examples/src/producer.ts +2 -2
- package/examples/src/replicator.ts +2 -1
- package/package.json +1 -1
- package/src/__snapshots__/cluster.test.ts.snap +266 -9
- package/src/api/fetch.ts +78 -28
- package/src/cluster.test.ts +8 -5
- package/src/cluster.ts +42 -37
- package/src/codecs/gzip.ts +9 -0
- package/src/codecs/index.ts +16 -0
- package/src/codecs/none.ts +6 -0
- package/src/codecs/types.ts +4 -0
- package/src/connection.ts +3 -2
- package/src/consumer/consumer-group.ts +8 -6
- package/src/consumer/consumer.ts +35 -24
- package/src/consumer/fetch-manager.ts +25 -19
- package/src/consumer/fetcher.ts +4 -4
- package/src/consumer/processor.ts +3 -1
- package/src/index.ts +2 -0
- package/src/producer/producer.ts +3 -3
- package/src/utils/api.ts +1 -1
- package/src/utils/decoder.ts +13 -3
- package/src/utils/logger.ts +37 -0
- package/src/utils/mutex.ts +31 -0
- package/src/utils/tracer.ts +6 -4
- package/src/utils/debug.ts +0 -9
package/src/cluster.ts
CHANGED
|
@@ -3,7 +3,8 @@ import { TLSSocketOptions } from 'tls';
|
|
|
3
3
|
import { API } from './api';
|
|
4
4
|
import { Broker, SASLProvider } from './broker';
|
|
5
5
|
import { SendRequest } from './connection';
|
|
6
|
-
import {
|
|
6
|
+
import { KafkaTSError } from './utils/error';
|
|
7
|
+
import { log } from './utils/logger';
|
|
7
8
|
|
|
8
9
|
type ClusterOptions = {
|
|
9
10
|
clientId: string | null;
|
|
@@ -13,73 +14,77 @@ type ClusterOptions = {
|
|
|
13
14
|
};
|
|
14
15
|
|
|
15
16
|
export class Cluster {
|
|
16
|
-
private seedBroker:
|
|
17
|
-
private
|
|
17
|
+
private seedBroker = new Broker({ clientId: null, sasl: null, ssl: null, options: { port: 9092 } });
|
|
18
|
+
private brokers: { nodeId: number; broker: Broker }[] = [];
|
|
19
|
+
private brokerMetadata: Record<number, Awaited<ReturnType<(typeof API.METADATA)['response']>>['brokers'][number]> =
|
|
20
|
+
{};
|
|
18
21
|
|
|
19
|
-
constructor(private options: ClusterOptions) {
|
|
20
|
-
this.seedBroker = new Broker({
|
|
21
|
-
clientId: this.options.clientId,
|
|
22
|
-
sasl: this.options.sasl,
|
|
23
|
-
ssl: this.options.ssl,
|
|
24
|
-
options: this.options.bootstrapServers[0],
|
|
25
|
-
});
|
|
26
|
-
}
|
|
22
|
+
constructor(private options: ClusterOptions) {}
|
|
27
23
|
|
|
28
24
|
public async connect() {
|
|
29
|
-
await this.
|
|
25
|
+
this.seedBroker = await this.findSeedBroker();
|
|
26
|
+
|
|
30
27
|
const metadata = await this.sendRequest(API.METADATA, {
|
|
31
28
|
allowTopicAutoCreation: false,
|
|
32
29
|
includeTopicAuthorizedOperations: false,
|
|
33
30
|
topics: [],
|
|
34
31
|
});
|
|
35
|
-
|
|
36
|
-
this.brokerById = Object.fromEntries(
|
|
37
|
-
metadata.brokers.map(({ nodeId, ...options }) => [
|
|
38
|
-
nodeId,
|
|
39
|
-
new Broker({
|
|
40
|
-
clientId: this.options.clientId,
|
|
41
|
-
sasl: this.options.sasl,
|
|
42
|
-
ssl: this.options.ssl,
|
|
43
|
-
options,
|
|
44
|
-
}),
|
|
45
|
-
]),
|
|
46
|
-
);
|
|
47
|
-
return this;
|
|
32
|
+
this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
|
|
48
33
|
}
|
|
49
34
|
|
|
50
35
|
public async disconnect() {
|
|
51
|
-
await Promise.all(
|
|
52
|
-
this.seedBroker.disconnect(),
|
|
53
|
-
...Object.values(this.brokerById).map((broker) => broker.disconnect()),
|
|
54
|
-
]);
|
|
36
|
+
await Promise.all(this.brokers.map((x) => x.broker.disconnect()));
|
|
55
37
|
}
|
|
56
38
|
|
|
39
|
+
public setSeedBroker = async (nodeId: number) => {
|
|
40
|
+
await this.releaseBroker(this.seedBroker);
|
|
41
|
+
this.seedBroker = await this.acquireBroker(nodeId);
|
|
42
|
+
};
|
|
43
|
+
|
|
57
44
|
public sendRequest: SendRequest = (...args) => this.seedBroker.sendRequest(...args);
|
|
58
45
|
|
|
59
46
|
public sendRequestToNode =
|
|
60
47
|
(nodeId: number): SendRequest =>
|
|
61
48
|
async (...args) => {
|
|
62
|
-
|
|
49
|
+
let broker = this.brokers.find((x) => x.nodeId === nodeId)?.broker;
|
|
63
50
|
if (!broker) {
|
|
64
|
-
|
|
51
|
+
broker = await this.acquireBroker(nodeId);
|
|
65
52
|
}
|
|
66
|
-
await broker.ensureConnected();
|
|
67
53
|
return broker.sendRequest(...args);
|
|
68
54
|
};
|
|
69
55
|
|
|
70
|
-
|
|
56
|
+
public async acquireBroker(nodeId: number) {
|
|
57
|
+
const broker = new Broker({
|
|
58
|
+
clientId: this.options.clientId,
|
|
59
|
+
sasl: this.options.sasl,
|
|
60
|
+
ssl: this.options.ssl,
|
|
61
|
+
options: this.brokerMetadata[nodeId],
|
|
62
|
+
});
|
|
63
|
+
this.brokers.push({ nodeId, broker });
|
|
64
|
+
await broker.connect();
|
|
65
|
+
return broker;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
public async releaseBroker(broker: Broker) {
|
|
69
|
+
await broker.disconnect();
|
|
70
|
+
this.brokers = this.brokers.filter((x) => x.broker !== broker);
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
private async findSeedBroker() {
|
|
71
74
|
const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
|
|
72
75
|
for (const options of randomizedBrokers) {
|
|
73
76
|
try {
|
|
74
|
-
|
|
77
|
+
const broker = await new Broker({
|
|
75
78
|
clientId: this.options.clientId,
|
|
76
79
|
sasl: this.options.sasl,
|
|
77
80
|
ssl: this.options.ssl,
|
|
78
81
|
options,
|
|
79
|
-
})
|
|
80
|
-
|
|
82
|
+
});
|
|
83
|
+
await broker.connect();
|
|
84
|
+
this.brokers.push({ nodeId: -1, broker });
|
|
85
|
+
return broker;
|
|
81
86
|
} catch (error) {
|
|
82
|
-
|
|
87
|
+
log.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
|
|
83
88
|
}
|
|
84
89
|
}
|
|
85
90
|
throw new KafkaTSError('No seed brokers found');
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { gzip, unzip } from 'zlib';
|
|
2
|
+
import { Codec } from './types';
|
|
3
|
+
|
|
4
|
+
export const GZIP: Codec = {
|
|
5
|
+
compress: async (data) =>
|
|
6
|
+
new Promise<Buffer>((resolve, reject) => gzip(data, (err, result) => (err ? reject(err) : resolve(result)))),
|
|
7
|
+
decompress: async (data) =>
|
|
8
|
+
new Promise<Buffer>((resolve, reject) => unzip(data, (err, result) => (err ? reject(err) : resolve(result)))),
|
|
9
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { GZIP } from './gzip';
|
|
2
|
+
import { NONE } from './none';
|
|
3
|
+
import { Codec } from './types';
|
|
4
|
+
|
|
5
|
+
const codecs: Record<number, Codec> = {
|
|
6
|
+
0: NONE,
|
|
7
|
+
1: GZIP,
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export const findCodec = (type: number) => {
|
|
11
|
+
const codec = codecs[type];
|
|
12
|
+
if (!codec) {
|
|
13
|
+
throw new Error(`Unsupported codec: ${type}`);
|
|
14
|
+
}
|
|
15
|
+
return codec;
|
|
16
|
+
};
|
package/src/connection.ts
CHANGED
|
@@ -59,6 +59,7 @@ export class Connection {
|
|
|
59
59
|
});
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
+
@trace()
|
|
62
63
|
public disconnect() {
|
|
63
64
|
this.socket.removeAllListeners();
|
|
64
65
|
return new Promise<void>((resolve) => {
|
|
@@ -69,7 +70,7 @@ export class Connection {
|
|
|
69
70
|
});
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
@trace((api, body) => ({
|
|
73
|
+
@trace((api, body) => ({ message: getApiName(api), body }))
|
|
73
74
|
public async sendRequest<Request, Response>(api: Api<Request, Response>, body: Request): Promise<Response> {
|
|
74
75
|
const correlationId = this.nextCorrelationId();
|
|
75
76
|
|
|
@@ -90,7 +91,7 @@ export class Connection {
|
|
|
90
91
|
reject(error);
|
|
91
92
|
}
|
|
92
93
|
});
|
|
93
|
-
const response = api.response(responseDecoder);
|
|
94
|
+
const response = await api.response(responseDecoder);
|
|
94
95
|
|
|
95
96
|
assert(
|
|
96
97
|
responseDecoder.getOffset() - 4 === responseSize,
|
|
@@ -30,6 +30,8 @@ export class ConsumerGroup {
|
|
|
30
30
|
|
|
31
31
|
public async join() {
|
|
32
32
|
await this.findCoordinator();
|
|
33
|
+
await this.options.cluster.setSeedBroker(this.coordinatorId);
|
|
34
|
+
|
|
33
35
|
await this.joinGroup();
|
|
34
36
|
await this.syncGroup();
|
|
35
37
|
await this.offsetFetch();
|
|
@@ -70,7 +72,7 @@ export class ConsumerGroup {
|
|
|
70
72
|
private async joinGroup(): Promise<void> {
|
|
71
73
|
const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
|
|
72
74
|
try {
|
|
73
|
-
const response = await cluster.
|
|
75
|
+
const response = await cluster.sendRequest(API.JOIN_GROUP, {
|
|
74
76
|
groupId,
|
|
75
77
|
groupInstanceId,
|
|
76
78
|
memberId: this.memberId,
|
|
@@ -113,7 +115,7 @@ export class ConsumerGroup {
|
|
|
113
115
|
assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
|
|
114
116
|
}
|
|
115
117
|
|
|
116
|
-
const response = await cluster.
|
|
118
|
+
const response = await cluster.sendRequest(API.SYNC_GROUP, {
|
|
117
119
|
groupId,
|
|
118
120
|
groupInstanceId,
|
|
119
121
|
memberId: this.memberId,
|
|
@@ -144,7 +146,7 @@ export class ConsumerGroup {
|
|
|
144
146
|
};
|
|
145
147
|
if (!request.groups.length) return;
|
|
146
148
|
|
|
147
|
-
const response = await cluster.
|
|
149
|
+
const response = await cluster.sendRequest(API.OFFSET_FETCH, request);
|
|
148
150
|
response.groups.forEach((group) => {
|
|
149
151
|
group.topics.forEach((topic) => {
|
|
150
152
|
topic.partitions
|
|
@@ -177,13 +179,13 @@ export class ConsumerGroup {
|
|
|
177
179
|
if (!request.topics.length) {
|
|
178
180
|
return;
|
|
179
181
|
}
|
|
180
|
-
await cluster.
|
|
182
|
+
await cluster.sendRequest(API.OFFSET_COMMIT, request);
|
|
181
183
|
offsetManager.flush();
|
|
182
184
|
}
|
|
183
185
|
|
|
184
186
|
public async heartbeat() {
|
|
185
187
|
const { cluster, groupId, groupInstanceId } = this.options;
|
|
186
|
-
await cluster.
|
|
188
|
+
await cluster.sendRequest(API.HEARTBEAT, {
|
|
187
189
|
groupId,
|
|
188
190
|
groupInstanceId,
|
|
189
191
|
memberId: this.memberId,
|
|
@@ -195,7 +197,7 @@ export class ConsumerGroup {
|
|
|
195
197
|
const { cluster, groupId, groupInstanceId } = this.options;
|
|
196
198
|
this.stopHeartbeater();
|
|
197
199
|
try {
|
|
198
|
-
await cluster.
|
|
200
|
+
await cluster.sendRequest(API.LEAVE_GROUP, {
|
|
199
201
|
groupId,
|
|
200
202
|
members: [{ memberId: this.memberId, groupInstanceId, reason: null }],
|
|
201
203
|
});
|
package/src/consumer/consumer.ts
CHANGED
|
@@ -2,16 +2,19 @@ import { API, API_ERROR } from '../api';
|
|
|
2
2
|
import { IsolationLevel } from '../api/fetch';
|
|
3
3
|
import { Assignment } from '../api/sync-group';
|
|
4
4
|
import { Cluster } from '../cluster';
|
|
5
|
-
import {
|
|
5
|
+
import { distributeMessagesToTopicPartitionLeaders } from '../distributors/messages-to-topic-partition-leaders';
|
|
6
6
|
import { Message } from '../types';
|
|
7
7
|
import { delay } from '../utils/delay';
|
|
8
8
|
import { ConnectionError, KafkaTSApiError } from '../utils/error';
|
|
9
|
-
import {
|
|
9
|
+
import { log } from '../utils/logger';
|
|
10
|
+
import { createTracer } from '../utils/tracer';
|
|
10
11
|
import { ConsumerGroup } from './consumer-group';
|
|
11
12
|
import { ConsumerMetadata } from './consumer-metadata';
|
|
12
|
-
import {
|
|
13
|
+
import { BatchGranularity, FetchManager } from './fetch-manager';
|
|
13
14
|
import { OffsetManager } from './offset-manager';
|
|
14
15
|
|
|
16
|
+
const trace = createTracer('Consumer');
|
|
17
|
+
|
|
15
18
|
export type ConsumerOptions = {
|
|
16
19
|
topics: string[];
|
|
17
20
|
groupId?: string | null;
|
|
@@ -26,7 +29,6 @@ export type ConsumerOptions = {
|
|
|
26
29
|
partitionMaxBytes?: number;
|
|
27
30
|
allowTopicAutoCreation?: boolean;
|
|
28
31
|
fromBeginning?: boolean;
|
|
29
|
-
retrier?: Retrier;
|
|
30
32
|
batchGranularity?: BatchGranularity;
|
|
31
33
|
concurrency?: number;
|
|
32
34
|
} & ({ onBatch: (messages: Required<Message>[]) => unknown } | { onMessage: (message: Required<Message>) => unknown });
|
|
@@ -57,7 +59,6 @@ export class Consumer {
|
|
|
57
59
|
isolationLevel: options.isolationLevel ?? IsolationLevel.READ_UNCOMMITTED,
|
|
58
60
|
allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
|
|
59
61
|
fromBeginning: options.fromBeginning ?? false,
|
|
60
|
-
retrier: options.retrier ?? defaultRetrier,
|
|
61
62
|
batchGranularity: options.batchGranularity ?? 'partition',
|
|
62
63
|
concurrency: options.concurrency ?? 1,
|
|
63
64
|
};
|
|
@@ -94,16 +95,17 @@ export class Consumer {
|
|
|
94
95
|
await this.offsetManager.fetchOffsets({ fromBeginning });
|
|
95
96
|
await this.consumerGroup?.join();
|
|
96
97
|
} catch (error) {
|
|
97
|
-
|
|
98
|
-
|
|
98
|
+
log.error('Failed to start consumer', error);
|
|
99
|
+
log.debug(`Restarting consumer in 1 second...`);
|
|
99
100
|
await delay(1000);
|
|
100
101
|
|
|
101
102
|
if (this.stopHook) return (this.stopHook as () => void)();
|
|
102
103
|
return this.close(true).then(() => this.start());
|
|
103
104
|
}
|
|
104
|
-
|
|
105
|
+
this.startFetchManager();
|
|
105
106
|
}
|
|
106
107
|
|
|
108
|
+
@trace()
|
|
107
109
|
public async close(force = false): Promise<void> {
|
|
108
110
|
if (!force) {
|
|
109
111
|
await new Promise<void>(async (resolve) => {
|
|
@@ -111,22 +113,31 @@ export class Consumer {
|
|
|
111
113
|
await this.fetchManager?.stop();
|
|
112
114
|
});
|
|
113
115
|
}
|
|
114
|
-
await this.consumerGroup
|
|
115
|
-
|
|
116
|
-
.catch((error) => console.warn(`Failed to leave group: ${error.message}`));
|
|
117
|
-
await this.cluster.disconnect().catch((error) => console.warn(`Failed to disconnect: ${error.message}`));
|
|
116
|
+
await this.consumerGroup?.leaveGroup().catch((error) => log.warn(`Failed to leave group: ${error.message}`));
|
|
117
|
+
await this.cluster.disconnect().catch((error) => log.warn(`Failed to disconnect: ${error.message}`));
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
private startFetchManager = async () => {
|
|
121
121
|
const { batchGranularity, concurrency } = this.options;
|
|
122
122
|
|
|
123
123
|
while (!this.stopHook) {
|
|
124
|
+
// TODO: If leader is not available, find another read replica
|
|
124
125
|
const nodeAssignments = Object.entries(
|
|
125
|
-
|
|
126
|
-
this.metadata.getAssignment(),
|
|
127
|
-
|
|
126
|
+
distributeMessagesToTopicPartitionLeaders(
|
|
127
|
+
Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) =>
|
|
128
|
+
partitions.map((partition) => ({ topic, partition })),
|
|
129
|
+
),
|
|
130
|
+
this.metadata.getTopicPartitionLeaderIds(),
|
|
131
|
+
),
|
|
132
|
+
).map(([nodeId, assignment]) => ({
|
|
133
|
+
nodeId: parseInt(nodeId),
|
|
134
|
+
assignment: Object.fromEntries(
|
|
135
|
+
Object.entries(assignment).map(([topic, partitions]) => [
|
|
136
|
+
topic,
|
|
137
|
+
Object.keys(partitions).map(Number),
|
|
138
|
+
]),
|
|
128
139
|
),
|
|
129
|
-
|
|
140
|
+
}));
|
|
130
141
|
|
|
131
142
|
const numPartitions = Object.values(this.metadata.getAssignment()).flat().length;
|
|
132
143
|
const numProcessors = Math.min(concurrency, numPartitions);
|
|
@@ -145,7 +156,7 @@ export class Consumer {
|
|
|
145
156
|
await this.fetchManager.start();
|
|
146
157
|
|
|
147
158
|
if (!nodeAssignments.length) {
|
|
148
|
-
|
|
159
|
+
log.debug('No partitions assigned. Waiting for reassignment...');
|
|
149
160
|
await delay(this.options.maxWaitMs);
|
|
150
161
|
await this.consumerGroup?.handleLastHeartbeat();
|
|
151
162
|
}
|
|
@@ -153,11 +164,11 @@ export class Consumer {
|
|
|
153
164
|
await this.fetchManager.stop();
|
|
154
165
|
|
|
155
166
|
if ((error as KafkaTSApiError).errorCode === API_ERROR.REBALANCE_IN_PROGRESS) {
|
|
156
|
-
|
|
167
|
+
log.debug('Rebalance in progress...');
|
|
157
168
|
continue;
|
|
158
169
|
}
|
|
159
170
|
if ((error as KafkaTSApiError).errorCode === API_ERROR.FENCED_INSTANCE_ID) {
|
|
160
|
-
|
|
171
|
+
log.debug('New consumer with the same groupInstanceId joined. Exiting the consumer...');
|
|
161
172
|
this.close();
|
|
162
173
|
break;
|
|
163
174
|
}
|
|
@@ -165,11 +176,11 @@ export class Consumer {
|
|
|
165
176
|
error instanceof ConnectionError ||
|
|
166
177
|
(error instanceof KafkaTSApiError && error.errorCode === API_ERROR.NOT_COORDINATOR)
|
|
167
178
|
) {
|
|
168
|
-
|
|
179
|
+
log.debug(`${error.message}. Restarting consumer...`);
|
|
169
180
|
this.close().then(() => this.start());
|
|
170
181
|
break;
|
|
171
182
|
}
|
|
172
|
-
|
|
183
|
+
log.error((error as Error).message, error);
|
|
173
184
|
this.close();
|
|
174
185
|
break;
|
|
175
186
|
}
|
|
@@ -177,12 +188,12 @@ export class Consumer {
|
|
|
177
188
|
this.stopHook?.();
|
|
178
189
|
};
|
|
179
190
|
|
|
191
|
+
@trace()
|
|
180
192
|
private async process(messages: Required<Message>[]) {
|
|
181
193
|
const { options } = this;
|
|
182
|
-
const { retrier } = options;
|
|
183
194
|
|
|
184
195
|
if ('onBatch' in options) {
|
|
185
|
-
await
|
|
196
|
+
await options.onBatch(messages);
|
|
186
197
|
|
|
187
198
|
messages.forEach(({ topic, partition, offset }) =>
|
|
188
199
|
this.offsetManager.resolve(topic, partition, offset + 1n),
|
|
@@ -190,7 +201,7 @@ export class Consumer {
|
|
|
190
201
|
} else if ('onMessage' in options) {
|
|
191
202
|
try {
|
|
192
203
|
for (const message of messages) {
|
|
193
|
-
await
|
|
204
|
+
await options.onMessage(message);
|
|
194
205
|
|
|
195
206
|
const { topic, partition, offset } = message;
|
|
196
207
|
this.offsetManager.resolve(topic, partition, offset + 1n);
|
|
@@ -14,7 +14,7 @@ const trace = createTracer('FetchManager');
|
|
|
14
14
|
export type BatchGranularity = 'partition' | 'topic' | 'broker';
|
|
15
15
|
|
|
16
16
|
type FetchManagerOptions = {
|
|
17
|
-
fetch: (nodeId: number, assignment: Assignment) => Promise<ReturnType<(typeof API.FETCH)['response']
|
|
17
|
+
fetch: (nodeId: number, assignment: Assignment) => Promise<Awaited<ReturnType<(typeof API.FETCH)['response']>>>;
|
|
18
18
|
process: (batch: Batch) => Promise<void>;
|
|
19
19
|
metadata: Metadata;
|
|
20
20
|
consumerGroup?: ConsumerGroup;
|
|
@@ -86,6 +86,7 @@ export class FetchManager extends EventEmitter<{ data: []; checkpoint: [number];
|
|
|
86
86
|
|
|
87
87
|
const batch = this.queue.shift();
|
|
88
88
|
if (!batch) {
|
|
89
|
+
// wait until new data is available or fetch manager is requested to stop
|
|
89
90
|
await new Promise<void>((resolve) => {
|
|
90
91
|
const onData = () => {
|
|
91
92
|
this.removeListener('stop', onStop);
|
|
@@ -109,35 +110,40 @@ export class FetchManager extends EventEmitter<{ data: []; checkpoint: [number];
|
|
|
109
110
|
return batch as Exclude<Entry, Checkpoint>;
|
|
110
111
|
}
|
|
111
112
|
|
|
112
|
-
private async onResponse(fetcherId: number, response: ReturnType<(typeof API.FETCH)['response']
|
|
113
|
+
private async onResponse(fetcherId: number, response: Awaited<ReturnType<(typeof API.FETCH)['response']>>) {
|
|
113
114
|
const { metadata, batchGranularity } = this.options;
|
|
114
115
|
|
|
115
116
|
const batches = fetchResponseToBatches(response, batchGranularity, metadata);
|
|
116
|
-
if (batches.length) {
|
|
117
|
+
if (!batches.length) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// wait until all broker batches have been processed or fetch manager is requested to stop
|
|
122
|
+
await new Promise<void>((resolve) => {
|
|
123
|
+
const onCheckpoint = (id: number) => {
|
|
124
|
+
if (id === fetcherId) {
|
|
125
|
+
this.removeListener('checkpoint', onCheckpoint);
|
|
126
|
+
this.removeListener('stop', onStop);
|
|
127
|
+
resolve();
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
const onStop = () => {
|
|
131
|
+
this.removeListener('checkpoint', onCheckpoint);
|
|
132
|
+
resolve();
|
|
133
|
+
};
|
|
134
|
+
this.on('checkpoint', onCheckpoint);
|
|
135
|
+
this.once('stop', onStop);
|
|
136
|
+
|
|
117
137
|
this.queue.push(...batches);
|
|
118
138
|
this.queue.push({ kind: 'checkpoint', fetcherId });
|
|
119
139
|
|
|
120
140
|
this.emit('data');
|
|
121
|
-
|
|
122
|
-
const onCheckpoint = (id: number) => {
|
|
123
|
-
if (id === fetcherId) {
|
|
124
|
-
this.removeListener('stop', onStop);
|
|
125
|
-
resolve();
|
|
126
|
-
}
|
|
127
|
-
};
|
|
128
|
-
const onStop = () => {
|
|
129
|
-
this.removeListener('checkpoint', onCheckpoint);
|
|
130
|
-
resolve();
|
|
131
|
-
};
|
|
132
|
-
this.once('checkpoint', onCheckpoint);
|
|
133
|
-
this.once('stop', onStop);
|
|
134
|
-
});
|
|
135
|
-
}
|
|
141
|
+
});
|
|
136
142
|
}
|
|
137
143
|
}
|
|
138
144
|
|
|
139
145
|
const fetchResponseToBatches = (
|
|
140
|
-
batch: ReturnType<typeof API.FETCH.response
|
|
146
|
+
batch: Awaited<ReturnType<typeof API.FETCH.response>>,
|
|
141
147
|
batchGranularity: BatchGranularity,
|
|
142
148
|
metadata: Metadata,
|
|
143
149
|
): Batch[] => {
|
package/src/consumer/fetcher.ts
CHANGED
|
@@ -10,8 +10,8 @@ type FetcherOptions = {
|
|
|
10
10
|
nodeId: number;
|
|
11
11
|
assignment: Assignment;
|
|
12
12
|
consumerGroup?: ConsumerGroup;
|
|
13
|
-
fetch: (nodeId: number, assignment: Assignment) => Promise<ReturnType<(typeof API.FETCH)['response']
|
|
14
|
-
onResponse: (fetcherId: number, response: ReturnType<(typeof API.FETCH)['response']
|
|
13
|
+
fetch: (nodeId: number, assignment: Assignment) => Promise<Awaited<ReturnType<(typeof API.FETCH)['response']>>>;
|
|
14
|
+
onResponse: (fetcherId: number, response: Awaited<ReturnType<(typeof API.FETCH)['response']>>) => Promise<void>;
|
|
15
15
|
};
|
|
16
16
|
|
|
17
17
|
export class Fetcher extends EventEmitter<{ stop: []; stopped: []; data: []; drain: [] }> {
|
|
@@ -26,10 +26,10 @@ export class Fetcher extends EventEmitter<{ stop: []; stopped: []; data: []; dra
|
|
|
26
26
|
|
|
27
27
|
public async loop() {
|
|
28
28
|
const { nodeId, assignment, consumerGroup, fetch, onResponse } = this.options;
|
|
29
|
-
|
|
29
|
+
|
|
30
30
|
this.isRunning = true;
|
|
31
31
|
this.once('stop', () => (this.isRunning = false));
|
|
32
|
-
|
|
32
|
+
|
|
33
33
|
try {
|
|
34
34
|
while (this.isRunning) {
|
|
35
35
|
const response = await fetch(nodeId, assignment);
|
|
@@ -25,7 +25,9 @@ export class Processor extends EventEmitter<{ stop: []; stopped: [] }> {
|
|
|
25
25
|
try {
|
|
26
26
|
while (this.isRunning) {
|
|
27
27
|
const batch = await poll();
|
|
28
|
-
|
|
28
|
+
if (batch.length) {
|
|
29
|
+
await process(batch);
|
|
30
|
+
}
|
|
29
31
|
}
|
|
30
32
|
} finally {
|
|
31
33
|
this.isRunning = false;
|
package/src/index.ts
CHANGED
package/src/producer/producer.ts
CHANGED
|
@@ -34,7 +34,7 @@ export class Producer {
|
|
|
34
34
|
this.partition = this.options.partitioner({ metadata: this.metadata });
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
public async send(messages: Message[]) {
|
|
37
|
+
public async send(messages: Message[], { acks = -1 }: { acks?: -1 | 1 } = {}) {
|
|
38
38
|
await this.ensureConnected();
|
|
39
39
|
|
|
40
40
|
const { allowTopicAutoCreation } = this.options;
|
|
@@ -44,7 +44,7 @@ export class Producer {
|
|
|
44
44
|
await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
|
|
45
45
|
|
|
46
46
|
const nodeTopicPartitionMessages = distributeMessagesToTopicPartitionLeaders(
|
|
47
|
-
messages.map(message => ({ ...message, partition: this.partition(message) })),
|
|
47
|
+
messages.map((message) => ({ ...message, partition: this.partition(message) })),
|
|
48
48
|
this.metadata.getTopicPartitionLeaderIds(),
|
|
49
49
|
);
|
|
50
50
|
|
|
@@ -52,7 +52,7 @@ export class Producer {
|
|
|
52
52
|
Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
|
|
53
53
|
await this.cluster.sendRequestToNode(parseInt(nodeId))(API.PRODUCE, {
|
|
54
54
|
transactionalId: null,
|
|
55
|
-
acks
|
|
55
|
+
acks,
|
|
56
56
|
timeoutMs: 5000,
|
|
57
57
|
topicData: Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
|
|
58
58
|
name: topic,
|
package/src/utils/api.ts
CHANGED
|
@@ -5,7 +5,7 @@ export type Api<Request, Response> = {
|
|
|
5
5
|
apiKey: number;
|
|
6
6
|
apiVersion: number;
|
|
7
7
|
request: (encoder: Encoder, body: Request) => Encoder;
|
|
8
|
-
response: (buffer: Decoder) => Response;
|
|
8
|
+
response: (buffer: Decoder) => Promise<Response> | Response;
|
|
9
9
|
};
|
|
10
10
|
|
|
11
11
|
export const createApi = <Request, Response>(api: Api<Request, Response>) => api;
|
package/src/utils/decoder.ts
CHANGED
|
@@ -7,6 +7,10 @@ export class Decoder {
|
|
|
7
7
|
return this.offset;
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
+
public getBuffer() {
|
|
11
|
+
return this.buffer;
|
|
12
|
+
}
|
|
13
|
+
|
|
10
14
|
public getBufferLength() {
|
|
11
15
|
return this.buffer.length;
|
|
12
16
|
}
|
|
@@ -132,6 +136,12 @@ export class Decoder {
|
|
|
132
136
|
return results;
|
|
133
137
|
}
|
|
134
138
|
|
|
139
|
+
public readVarIntArray<T>(callback: (opts: Decoder) => T): T[] {
|
|
140
|
+
const length = this.readVarInt();
|
|
141
|
+
const results = Array.from({ length }).map(() => callback(this));
|
|
142
|
+
return results;
|
|
143
|
+
}
|
|
144
|
+
|
|
135
145
|
public readRecords<T>(callback: (opts: Decoder) => T): T[] {
|
|
136
146
|
const length = this.readInt32();
|
|
137
147
|
|
|
@@ -143,9 +153,9 @@ export class Decoder {
|
|
|
143
153
|
});
|
|
144
154
|
}
|
|
145
155
|
|
|
146
|
-
public read(length
|
|
147
|
-
const value = this.buffer.subarray(this.offset, this.offset + length);
|
|
148
|
-
this.offset +=
|
|
156
|
+
public read(length?: number) {
|
|
157
|
+
const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);
|
|
158
|
+
this.offset += Buffer.byteLength(value);
|
|
149
159
|
return value;
|
|
150
160
|
}
|
|
151
161
|
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export interface Logger {
|
|
2
|
+
debug: (message: string, metadata?: unknown) => void;
|
|
3
|
+
info: (message: string, metadata?: unknown) => void;
|
|
4
|
+
warn: (message: string, metadata?: unknown) => void;
|
|
5
|
+
error: (message: string, metadata?: unknown) => void;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export const jsonSerializer = (_: unknown, v: unknown) => {
|
|
9
|
+
if (v instanceof Error) {
|
|
10
|
+
return { name: v.name, message: v.message, stack: v.stack, cause: v.cause };
|
|
11
|
+
}
|
|
12
|
+
if (typeof v === 'bigint') {
|
|
13
|
+
return v.toString();
|
|
14
|
+
}
|
|
15
|
+
return v;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
class JsonLogger implements Logger {
|
|
19
|
+
debug(message: string, metadata?: unknown) {
|
|
20
|
+
console.debug(JSON.stringify({ message, metadata }, jsonSerializer));
|
|
21
|
+
}
|
|
22
|
+
info(message: string, metadata?: unknown) {
|
|
23
|
+
console.info(JSON.stringify({ message, metadata }, jsonSerializer));
|
|
24
|
+
}
|
|
25
|
+
warn(message: string, metadata?: unknown) {
|
|
26
|
+
console.warn(JSON.stringify({ message, metadata }, jsonSerializer));
|
|
27
|
+
}
|
|
28
|
+
error(message: string, metadata?: unknown) {
|
|
29
|
+
console.error(JSON.stringify({ message, metadata }, jsonSerializer));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export let log: Logger = new JsonLogger();
|
|
34
|
+
|
|
35
|
+
export const setLogger = (newLogger: Logger) => {
|
|
36
|
+
log = newLogger;
|
|
37
|
+
};
|