kafka-ts 0.0.1-beta.3 → 0.0.1-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cluster.ts CHANGED
@@ -3,7 +3,8 @@ import { TLSSocketOptions } from 'tls';
3
3
  import { API } from './api';
4
4
  import { Broker, SASLProvider } from './broker';
5
5
  import { SendRequest } from './connection';
6
- import { ConnectionError, KafkaTSError } from './utils/error';
6
+ import { KafkaTSError } from './utils/error';
7
+ import { log } from './utils/logger';
7
8
 
8
9
  type ClusterOptions = {
9
10
  clientId: string | null;
@@ -13,73 +14,77 @@ type ClusterOptions = {
13
14
  };
14
15
 
15
16
  export class Cluster {
16
- private seedBroker: Broker;
17
- private brokerById: Record<number, Broker> = {};
17
+ private seedBroker = new Broker({ clientId: null, sasl: null, ssl: null, options: { port: 9092 } });
18
+ private brokers: { nodeId: number; broker: Broker }[] = [];
19
+ private brokerMetadata: Record<number, Awaited<ReturnType<(typeof API.METADATA)['response']>>['brokers'][number]> =
20
+ {};
18
21
 
19
- constructor(private options: ClusterOptions) {
20
- this.seedBroker = new Broker({
21
- clientId: this.options.clientId,
22
- sasl: this.options.sasl,
23
- ssl: this.options.ssl,
24
- options: this.options.bootstrapServers[0],
25
- });
26
- }
22
+ constructor(private options: ClusterOptions) {}
27
23
 
28
24
  public async connect() {
29
- await this.connectSeedBroker();
25
+ this.seedBroker = await this.findSeedBroker();
26
+
30
27
  const metadata = await this.sendRequest(API.METADATA, {
31
28
  allowTopicAutoCreation: false,
32
29
  includeTopicAuthorizedOperations: false,
33
30
  topics: [],
34
31
  });
35
-
36
- this.brokerById = Object.fromEntries(
37
- metadata.brokers.map(({ nodeId, ...options }) => [
38
- nodeId,
39
- new Broker({
40
- clientId: this.options.clientId,
41
- sasl: this.options.sasl,
42
- ssl: this.options.ssl,
43
- options,
44
- }),
45
- ]),
46
- );
47
- return this;
32
+ this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
48
33
  }
49
34
 
50
35
  public async disconnect() {
51
- await Promise.all([
52
- this.seedBroker.disconnect(),
53
- ...Object.values(this.brokerById).map((broker) => broker.disconnect()),
54
- ]);
36
+ await Promise.all(this.brokers.map((x) => x.broker.disconnect()));
55
37
  }
56
38
 
39
+ public setSeedBroker = async (nodeId: number) => {
40
+ await this.releaseBroker(this.seedBroker);
41
+ this.seedBroker = await this.acquireBroker(nodeId);
42
+ };
43
+
57
44
  public sendRequest: SendRequest = (...args) => this.seedBroker.sendRequest(...args);
58
45
 
59
46
  public sendRequestToNode =
60
47
  (nodeId: number): SendRequest =>
61
48
  async (...args) => {
62
- const broker = this.brokerById[nodeId];
49
+ let broker = this.brokers.find((x) => x.nodeId === nodeId)?.broker;
63
50
  if (!broker) {
64
- throw new ConnectionError(`Broker ${nodeId} is not available`);
51
+ broker = await this.acquireBroker(nodeId);
65
52
  }
66
- await broker.ensureConnected();
67
53
  return broker.sendRequest(...args);
68
54
  };
69
55
 
70
- private async connectSeedBroker() {
56
+ public async acquireBroker(nodeId: number) {
57
+ const broker = new Broker({
58
+ clientId: this.options.clientId,
59
+ sasl: this.options.sasl,
60
+ ssl: this.options.ssl,
61
+ options: this.brokerMetadata[nodeId],
62
+ });
63
+ this.brokers.push({ nodeId, broker });
64
+ await broker.connect();
65
+ return broker;
66
+ }
67
+
68
+ public async releaseBroker(broker: Broker) {
69
+ await broker.disconnect();
70
+ this.brokers = this.brokers.filter((x) => x.broker !== broker);
71
+ };
72
+
73
+ private async findSeedBroker() {
71
74
  const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
72
75
  for (const options of randomizedBrokers) {
73
76
  try {
74
- this.seedBroker = await new Broker({
77
+ const broker = await new Broker({
75
78
  clientId: this.options.clientId,
76
79
  sasl: this.options.sasl,
77
80
  ssl: this.options.ssl,
78
81
  options,
79
- }).connect();
80
- return;
82
+ });
83
+ await broker.connect();
84
+ this.brokers.push({ nodeId: -1, broker });
85
+ return broker;
81
86
  } catch (error) {
82
- console.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
87
+ log.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
83
88
  }
84
89
  }
85
90
  throw new KafkaTSError('No seed brokers found');
@@ -0,0 +1,9 @@
1
+ import { gzip, unzip } from 'zlib';
2
+ import { Codec } from './types';
3
+
4
+ export const GZIP: Codec = {
5
+ compress: async (data) =>
6
+ new Promise<Buffer>((resolve, reject) => gzip(data, (err, result) => (err ? reject(err) : resolve(result)))),
7
+ decompress: async (data) =>
8
+ new Promise<Buffer>((resolve, reject) => unzip(data, (err, result) => (err ? reject(err) : resolve(result)))),
9
+ };
@@ -0,0 +1,16 @@
1
+ import { GZIP } from './gzip';
2
+ import { NONE } from './none';
3
+ import { Codec } from './types';
4
+
5
+ const codecs: Record<number, Codec> = {
6
+ 0: NONE,
7
+ 1: GZIP,
8
+ };
9
+
10
+ export const findCodec = (type: number) => {
11
+ const codec = codecs[type];
12
+ if (!codec) {
13
+ throw new Error(`Unsupported codec: ${type}`);
14
+ }
15
+ return codec;
16
+ };
@@ -0,0 +1,6 @@
1
+ import { Codec } from './types';
2
+
3
+ export const NONE: Codec = {
4
+ compress: (data: Buffer) => Promise.resolve(data),
5
+ decompress: (data: Buffer) => Promise.resolve(data),
6
+ };
@@ -0,0 +1,4 @@
1
+ export type Codec = {
2
+ compress: (data: Buffer) => Promise<Buffer>;
3
+ decompress: (data: Buffer) => Promise<Buffer>;
4
+ };
package/src/connection.ts CHANGED
@@ -59,6 +59,7 @@ export class Connection {
59
59
  });
60
60
  }
61
61
 
62
+ @trace()
62
63
  public disconnect() {
63
64
  this.socket.removeAllListeners();
64
65
  return new Promise<void>((resolve) => {
@@ -69,7 +70,7 @@ export class Connection {
69
70
  });
70
71
  }
71
72
 
72
- @trace((api, body) => ({ apiName: getApiName(api), body }))
73
+ @trace((api, body) => ({ message: getApiName(api), body }))
73
74
  public async sendRequest<Request, Response>(api: Api<Request, Response>, body: Request): Promise<Response> {
74
75
  const correlationId = this.nextCorrelationId();
75
76
 
@@ -90,7 +91,7 @@ export class Connection {
90
91
  reject(error);
91
92
  }
92
93
  });
93
- const response = api.response(responseDecoder);
94
+ const response = await api.response(responseDecoder);
94
95
 
95
96
  assert(
96
97
  responseDecoder.getOffset() - 4 === responseSize,
@@ -30,6 +30,8 @@ export class ConsumerGroup {
30
30
 
31
31
  public async join() {
32
32
  await this.findCoordinator();
33
+ await this.options.cluster.setSeedBroker(this.coordinatorId);
34
+
33
35
  await this.joinGroup();
34
36
  await this.syncGroup();
35
37
  await this.offsetFetch();
@@ -70,7 +72,7 @@ export class ConsumerGroup {
70
72
  private async joinGroup(): Promise<void> {
71
73
  const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
72
74
  try {
73
- const response = await cluster.sendRequestToNode(this.coordinatorId)(API.JOIN_GROUP, {
75
+ const response = await cluster.sendRequest(API.JOIN_GROUP, {
74
76
  groupId,
75
77
  groupInstanceId,
76
78
  memberId: this.memberId,
@@ -113,7 +115,7 @@ export class ConsumerGroup {
113
115
  assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
114
116
  }
115
117
 
116
- const response = await cluster.sendRequestToNode(this.coordinatorId)(API.SYNC_GROUP, {
118
+ const response = await cluster.sendRequest(API.SYNC_GROUP, {
117
119
  groupId,
118
120
  groupInstanceId,
119
121
  memberId: this.memberId,
@@ -144,7 +146,7 @@ export class ConsumerGroup {
144
146
  };
145
147
  if (!request.groups.length) return;
146
148
 
147
- const response = await cluster.sendRequestToNode(this.coordinatorId)(API.OFFSET_FETCH, request);
149
+ const response = await cluster.sendRequest(API.OFFSET_FETCH, request);
148
150
  response.groups.forEach((group) => {
149
151
  group.topics.forEach((topic) => {
150
152
  topic.partitions
@@ -177,13 +179,13 @@ export class ConsumerGroup {
177
179
  if (!request.topics.length) {
178
180
  return;
179
181
  }
180
- await cluster.sendRequestToNode(this.coordinatorId)(API.OFFSET_COMMIT, request);
182
+ await cluster.sendRequest(API.OFFSET_COMMIT, request);
181
183
  offsetManager.flush();
182
184
  }
183
185
 
184
186
  public async heartbeat() {
185
187
  const { cluster, groupId, groupInstanceId } = this.options;
186
- await cluster.sendRequestToNode(this.coordinatorId)(API.HEARTBEAT, {
188
+ await cluster.sendRequest(API.HEARTBEAT, {
187
189
  groupId,
188
190
  groupInstanceId,
189
191
  memberId: this.memberId,
@@ -195,7 +197,7 @@ export class ConsumerGroup {
195
197
  const { cluster, groupId, groupInstanceId } = this.options;
196
198
  this.stopHeartbeater();
197
199
  try {
198
- await cluster.sendRequestToNode(this.coordinatorId)(API.LEAVE_GROUP, {
200
+ await cluster.sendRequest(API.LEAVE_GROUP, {
199
201
  groupId,
200
202
  members: [{ memberId: this.memberId, groupInstanceId, reason: null }],
201
203
  });
@@ -2,16 +2,19 @@ import { API, API_ERROR } from '../api';
2
2
  import { IsolationLevel } from '../api/fetch';
3
3
  import { Assignment } from '../api/sync-group';
4
4
  import { Cluster } from '../cluster';
5
- import { distributeAssignmentsToNodes } from '../distributors/assignments-to-replicas';
5
+ import { distributeMessagesToTopicPartitionLeaders } from '../distributors/messages-to-topic-partition-leaders';
6
6
  import { Message } from '../types';
7
7
  import { delay } from '../utils/delay';
8
8
  import { ConnectionError, KafkaTSApiError } from '../utils/error';
9
- import { defaultRetrier, Retrier } from '../utils/retrier';
9
+ import { log } from '../utils/logger';
10
+ import { createTracer } from '../utils/tracer';
10
11
  import { ConsumerGroup } from './consumer-group';
11
12
  import { ConsumerMetadata } from './consumer-metadata';
12
- import { FetchManager, BatchGranularity } from './fetch-manager';
13
+ import { BatchGranularity, FetchManager } from './fetch-manager';
13
14
  import { OffsetManager } from './offset-manager';
14
15
 
16
+ const trace = createTracer('Consumer');
17
+
15
18
  export type ConsumerOptions = {
16
19
  topics: string[];
17
20
  groupId?: string | null;
@@ -26,7 +29,6 @@ export type ConsumerOptions = {
26
29
  partitionMaxBytes?: number;
27
30
  allowTopicAutoCreation?: boolean;
28
31
  fromBeginning?: boolean;
29
- retrier?: Retrier;
30
32
  batchGranularity?: BatchGranularity;
31
33
  concurrency?: number;
32
34
  } & ({ onBatch: (messages: Required<Message>[]) => unknown } | { onMessage: (message: Required<Message>) => unknown });
@@ -57,7 +59,6 @@ export class Consumer {
57
59
  isolationLevel: options.isolationLevel ?? IsolationLevel.READ_UNCOMMITTED,
58
60
  allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
59
61
  fromBeginning: options.fromBeginning ?? false,
60
- retrier: options.retrier ?? defaultRetrier,
61
62
  batchGranularity: options.batchGranularity ?? 'partition',
62
63
  concurrency: options.concurrency ?? 1,
63
64
  };
@@ -94,16 +95,17 @@ export class Consumer {
94
95
  await this.offsetManager.fetchOffsets({ fromBeginning });
95
96
  await this.consumerGroup?.join();
96
97
  } catch (error) {
97
- console.error(error);
98
- console.debug(`Restarting consumer in 1 second...`);
98
+ log.error('Failed to start consumer', error);
99
+ log.debug(`Restarting consumer in 1 second...`);
99
100
  await delay(1000);
100
101
 
101
102
  if (this.stopHook) return (this.stopHook as () => void)();
102
103
  return this.close(true).then(() => this.start());
103
104
  }
104
- setImmediate(() => this.startFetchManager());
105
+ this.startFetchManager();
105
106
  }
106
107
 
108
+ @trace()
107
109
  public async close(force = false): Promise<void> {
108
110
  if (!force) {
109
111
  await new Promise<void>(async (resolve) => {
@@ -111,22 +113,31 @@ export class Consumer {
111
113
  await this.fetchManager?.stop();
112
114
  });
113
115
  }
114
- await this.consumerGroup
115
- ?.leaveGroup()
116
- .catch((error) => console.warn(`Failed to leave group: ${error.message}`));
117
- await this.cluster.disconnect().catch((error) => console.warn(`Failed to disconnect: ${error.message}`));
116
+ await this.consumerGroup?.leaveGroup().catch((error) => log.warn(`Failed to leave group: ${error.message}`));
117
+ await this.cluster.disconnect().catch((error) => log.warn(`Failed to disconnect: ${error.message}`));
118
118
  }
119
119
 
120
120
  private startFetchManager = async () => {
121
121
  const { batchGranularity, concurrency } = this.options;
122
122
 
123
123
  while (!this.stopHook) {
124
+ // TODO: If leader is not available, find another read replica
124
125
  const nodeAssignments = Object.entries(
125
- distributeAssignmentsToNodes(
126
- this.metadata.getAssignment(),
127
- this.metadata.getTopicPartitionReplicaIds(),
126
+ distributeMessagesToTopicPartitionLeaders(
127
+ Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) =>
128
+ partitions.map((partition) => ({ topic, partition })),
129
+ ),
130
+ this.metadata.getTopicPartitionLeaderIds(),
131
+ ),
132
+ ).map(([nodeId, assignment]) => ({
133
+ nodeId: parseInt(nodeId),
134
+ assignment: Object.fromEntries(
135
+ Object.entries(assignment).map(([topic, partitions]) => [
136
+ topic,
137
+ Object.keys(partitions).map(Number),
138
+ ]),
128
139
  ),
129
- ).map(([nodeId, assignment]) => ({ nodeId: parseInt(nodeId), assignment }));
140
+ }));
130
141
 
131
142
  const numPartitions = Object.values(this.metadata.getAssignment()).flat().length;
132
143
  const numProcessors = Math.min(concurrency, numPartitions);
@@ -145,7 +156,7 @@ export class Consumer {
145
156
  await this.fetchManager.start();
146
157
 
147
158
  if (!nodeAssignments.length) {
148
- console.debug('No partitions assigned. Waiting for reassignment...');
159
+ log.debug('No partitions assigned. Waiting for reassignment...');
149
160
  await delay(this.options.maxWaitMs);
150
161
  await this.consumerGroup?.handleLastHeartbeat();
151
162
  }
@@ -153,11 +164,11 @@ export class Consumer {
153
164
  await this.fetchManager.stop();
154
165
 
155
166
  if ((error as KafkaTSApiError).errorCode === API_ERROR.REBALANCE_IN_PROGRESS) {
156
- console.debug('Rebalance in progress...');
167
+ log.debug('Rebalance in progress...');
157
168
  continue;
158
169
  }
159
170
  if ((error as KafkaTSApiError).errorCode === API_ERROR.FENCED_INSTANCE_ID) {
160
- console.debug('New consumer with the same groupInstanceId joined. Exiting the consumer...');
171
+ log.debug('New consumer with the same groupInstanceId joined. Exiting the consumer...');
161
172
  this.close();
162
173
  break;
163
174
  }
@@ -165,11 +176,11 @@ export class Consumer {
165
176
  error instanceof ConnectionError ||
166
177
  (error instanceof KafkaTSApiError && error.errorCode === API_ERROR.NOT_COORDINATOR)
167
178
  ) {
168
- console.debug(`${error.message}. Restarting consumer...`);
179
+ log.debug(`${error.message}. Restarting consumer...`);
169
180
  this.close().then(() => this.start());
170
181
  break;
171
182
  }
172
- console.error(error);
183
+ log.error((error as Error).message, error);
173
184
  this.close();
174
185
  break;
175
186
  }
@@ -177,12 +188,12 @@ export class Consumer {
177
188
  this.stopHook?.();
178
189
  };
179
190
 
191
+ @trace()
180
192
  private async process(messages: Required<Message>[]) {
181
193
  const { options } = this;
182
- const { retrier } = options;
183
194
 
184
195
  if ('onBatch' in options) {
185
- await retrier(() => options.onBatch(messages));
196
+ await options.onBatch(messages);
186
197
 
187
198
  messages.forEach(({ topic, partition, offset }) =>
188
199
  this.offsetManager.resolve(topic, partition, offset + 1n),
@@ -190,7 +201,7 @@ export class Consumer {
190
201
  } else if ('onMessage' in options) {
191
202
  try {
192
203
  for (const message of messages) {
193
- await retrier(() => options.onMessage(message));
204
+ await options.onMessage(message);
194
205
 
195
206
  const { topic, partition, offset } = message;
196
207
  this.offsetManager.resolve(topic, partition, offset + 1n);
@@ -14,7 +14,7 @@ const trace = createTracer('FetchManager');
14
14
  export type BatchGranularity = 'partition' | 'topic' | 'broker';
15
15
 
16
16
  type FetchManagerOptions = {
17
- fetch: (nodeId: number, assignment: Assignment) => Promise<ReturnType<(typeof API.FETCH)['response']>>;
17
+ fetch: (nodeId: number, assignment: Assignment) => Promise<Awaited<ReturnType<(typeof API.FETCH)['response']>>>;
18
18
  process: (batch: Batch) => Promise<void>;
19
19
  metadata: Metadata;
20
20
  consumerGroup?: ConsumerGroup;
@@ -86,6 +86,7 @@ export class FetchManager extends EventEmitter<{ data: []; checkpoint: [number];
86
86
 
87
87
  const batch = this.queue.shift();
88
88
  if (!batch) {
89
+ // wait until new data is available or fetch manager is requested to stop
89
90
  await new Promise<void>((resolve) => {
90
91
  const onData = () => {
91
92
  this.removeListener('stop', onStop);
@@ -109,35 +110,40 @@ export class FetchManager extends EventEmitter<{ data: []; checkpoint: [number];
109
110
  return batch as Exclude<Entry, Checkpoint>;
110
111
  }
111
112
 
112
- private async onResponse(fetcherId: number, response: ReturnType<(typeof API.FETCH)['response']>) {
113
+ private async onResponse(fetcherId: number, response: Awaited<ReturnType<(typeof API.FETCH)['response']>>) {
113
114
  const { metadata, batchGranularity } = this.options;
114
115
 
115
116
  const batches = fetchResponseToBatches(response, batchGranularity, metadata);
116
- if (batches.length) {
117
+ if (!batches.length) {
118
+ return;
119
+ }
120
+
121
+ // wait until all broker batches have been processed or fetch manager is requested to stop
122
+ await new Promise<void>((resolve) => {
123
+ const onCheckpoint = (id: number) => {
124
+ if (id === fetcherId) {
125
+ this.removeListener('checkpoint', onCheckpoint);
126
+ this.removeListener('stop', onStop);
127
+ resolve();
128
+ }
129
+ };
130
+ const onStop = () => {
131
+ this.removeListener('checkpoint', onCheckpoint);
132
+ resolve();
133
+ };
134
+ this.on('checkpoint', onCheckpoint);
135
+ this.once('stop', onStop);
136
+
117
137
  this.queue.push(...batches);
118
138
  this.queue.push({ kind: 'checkpoint', fetcherId });
119
139
 
120
140
  this.emit('data');
121
- await new Promise<void>((resolve) => {
122
- const onCheckpoint = (id: number) => {
123
- if (id === fetcherId) {
124
- this.removeListener('stop', onStop);
125
- resolve();
126
- }
127
- };
128
- const onStop = () => {
129
- this.removeListener('checkpoint', onCheckpoint);
130
- resolve();
131
- };
132
- this.once('checkpoint', onCheckpoint);
133
- this.once('stop', onStop);
134
- });
135
- }
141
+ });
136
142
  }
137
143
  }
138
144
 
139
145
  const fetchResponseToBatches = (
140
- batch: ReturnType<typeof API.FETCH.response>,
146
+ batch: Awaited<ReturnType<typeof API.FETCH.response>>,
141
147
  batchGranularity: BatchGranularity,
142
148
  metadata: Metadata,
143
149
  ): Batch[] => {
@@ -10,8 +10,8 @@ type FetcherOptions = {
10
10
  nodeId: number;
11
11
  assignment: Assignment;
12
12
  consumerGroup?: ConsumerGroup;
13
- fetch: (nodeId: number, assignment: Assignment) => Promise<ReturnType<(typeof API.FETCH)['response']>>;
14
- onResponse: (fetcherId: number, response: ReturnType<(typeof API.FETCH)['response']>) => Promise<void>;
13
+ fetch: (nodeId: number, assignment: Assignment) => Promise<Awaited<ReturnType<(typeof API.FETCH)['response']>>>;
14
+ onResponse: (fetcherId: number, response: Awaited<ReturnType<(typeof API.FETCH)['response']>>) => Promise<void>;
15
15
  };
16
16
 
17
17
  export class Fetcher extends EventEmitter<{ stop: []; stopped: []; data: []; drain: [] }> {
@@ -26,10 +26,10 @@ export class Fetcher extends EventEmitter<{ stop: []; stopped: []; data: []; dra
26
26
 
27
27
  public async loop() {
28
28
  const { nodeId, assignment, consumerGroup, fetch, onResponse } = this.options;
29
-
29
+
30
30
  this.isRunning = true;
31
31
  this.once('stop', () => (this.isRunning = false));
32
-
32
+
33
33
  try {
34
34
  while (this.isRunning) {
35
35
  const response = await fetch(nodeId, assignment);
@@ -25,7 +25,9 @@ export class Processor extends EventEmitter<{ stop: []; stopped: [] }> {
25
25
  try {
26
26
  while (this.isRunning) {
27
27
  const batch = await poll();
28
- await process(batch);
28
+ if (batch.length) {
29
+ await process(batch);
30
+ }
29
31
  }
30
32
  } finally {
31
33
  this.isRunning = false;
package/src/index.ts CHANGED
@@ -5,3 +5,5 @@ export * from './client';
5
5
  export * from './distributors/partitioner';
6
6
  export * from './types';
7
7
  export * from './utils/error';
8
+ export * from './utils/logger';
9
+
@@ -34,7 +34,7 @@ export class Producer {
34
34
  this.partition = this.options.partitioner({ metadata: this.metadata });
35
35
  }
36
36
 
37
- public async send(messages: Message[]) {
37
+ public async send(messages: Message[], { acks = -1 }: { acks?: -1 | 1 } = {}) {
38
38
  await this.ensureConnected();
39
39
 
40
40
  const { allowTopicAutoCreation } = this.options;
@@ -44,7 +44,7 @@ export class Producer {
44
44
  await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
45
45
 
46
46
  const nodeTopicPartitionMessages = distributeMessagesToTopicPartitionLeaders(
47
- messages.map(message => ({ ...message, partition: this.partition(message) })),
47
+ messages.map((message) => ({ ...message, partition: this.partition(message) })),
48
48
  this.metadata.getTopicPartitionLeaderIds(),
49
49
  );
50
50
 
@@ -52,7 +52,7 @@ export class Producer {
52
52
  Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
53
53
  await this.cluster.sendRequestToNode(parseInt(nodeId))(API.PRODUCE, {
54
54
  transactionalId: null,
55
- acks: 1,
55
+ acks,
56
56
  timeoutMs: 5000,
57
57
  topicData: Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
58
58
  name: topic,
package/src/utils/api.ts CHANGED
@@ -5,7 +5,7 @@ export type Api<Request, Response> = {
5
5
  apiKey: number;
6
6
  apiVersion: number;
7
7
  request: (encoder: Encoder, body: Request) => Encoder;
8
- response: (buffer: Decoder) => Response;
8
+ response: (buffer: Decoder) => Promise<Response> | Response;
9
9
  };
10
10
 
11
11
  export const createApi = <Request, Response>(api: Api<Request, Response>) => api;
@@ -7,6 +7,10 @@ export class Decoder {
7
7
  return this.offset;
8
8
  }
9
9
 
10
+ public getBuffer() {
11
+ return this.buffer;
12
+ }
13
+
10
14
  public getBufferLength() {
11
15
  return this.buffer.length;
12
16
  }
@@ -132,6 +136,12 @@ export class Decoder {
132
136
  return results;
133
137
  }
134
138
 
139
+ public readVarIntArray<T>(callback: (opts: Decoder) => T): T[] {
140
+ const length = this.readVarInt();
141
+ const results = Array.from({ length }).map(() => callback(this));
142
+ return results;
143
+ }
144
+
135
145
  public readRecords<T>(callback: (opts: Decoder) => T): T[] {
136
146
  const length = this.readInt32();
137
147
 
@@ -143,9 +153,9 @@ export class Decoder {
143
153
  });
144
154
  }
145
155
 
146
- public read(length: number) {
147
- const value = this.buffer.subarray(this.offset, this.offset + length);
148
- this.offset += length;
156
+ public read(length?: number) {
157
+ const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);
158
+ this.offset += Buffer.byteLength(value);
149
159
  return value;
150
160
  }
151
161
 
@@ -0,0 +1,37 @@
1
+ export interface Logger {
2
+ debug: (message: string, metadata?: unknown) => void;
3
+ info: (message: string, metadata?: unknown) => void;
4
+ warn: (message: string, metadata?: unknown) => void;
5
+ error: (message: string, metadata?: unknown) => void;
6
+ }
7
+
8
+ export const jsonSerializer = (_: unknown, v: unknown) => {
9
+ if (v instanceof Error) {
10
+ return { name: v.name, message: v.message, stack: v.stack, cause: v.cause };
11
+ }
12
+ if (typeof v === 'bigint') {
13
+ return v.toString();
14
+ }
15
+ return v;
16
+ };
17
+
18
+ class JsonLogger implements Logger {
19
+ debug(message: string, metadata?: unknown) {
20
+ console.debug(JSON.stringify({ message, metadata }, jsonSerializer));
21
+ }
22
+ info(message: string, metadata?: unknown) {
23
+ console.info(JSON.stringify({ message, metadata }, jsonSerializer));
24
+ }
25
+ warn(message: string, metadata?: unknown) {
26
+ console.warn(JSON.stringify({ message, metadata }, jsonSerializer));
27
+ }
28
+ error(message: string, metadata?: unknown) {
29
+ console.error(JSON.stringify({ message, metadata }, jsonSerializer));
30
+ }
31
+ }
32
+
33
+ export let log: Logger = new JsonLogger();
34
+
35
+ export const setLogger = (newLogger: Logger) => {
36
+ log = newLogger;
37
+ };