@relaybus/kafka 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +215 -2
- package/dist/index.cjs +719 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +283 -0
- package/dist/index.d.ts +283 -0
- package/dist/index.js +711 -0
- package/dist/index.js.map +1 -0
- package/package.json +74 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,711 @@
|
|
|
1
|
+
import { RelayBusError, NonRetryableError, assertEventType, ConfigurationError, toErrorInfo, toMilliseconds } from '@relaybus/core';
|
|
2
|
+
|
|
3
|
+
// src/cloudevents-codec.ts
|
|
4
|
+
var InvalidKafkaMessageError = class extends RelayBusError {
|
|
5
|
+
};
|
|
6
|
+
var UnencodableEventError = class extends NonRetryableError {
|
|
7
|
+
name = "UnencodableEventError";
|
|
8
|
+
};
|
|
9
|
+
var SPEC_VERSION = "1.0";
|
|
10
|
+
var CONTENT_TYPE = "content-type";
|
|
11
|
+
var JSON_MEDIA_TYPE = "application/json";
|
|
12
|
+
var CLOUDEVENTS_PREFIX = "ce_";
|
|
13
|
+
var ESCAPE_PREFIX = "x-relaybus-h-";
|
|
14
|
+
var RESERVED_PREFIXES = [CLOUDEVENTS_PREFIX, "x-relaybus-"];
|
|
15
|
+
var HEADER = {
|
|
16
|
+
specVersion: "ce_specversion",
|
|
17
|
+
id: "ce_id",
|
|
18
|
+
type: "ce_type",
|
|
19
|
+
source: "ce_source",
|
|
20
|
+
time: "ce_time",
|
|
21
|
+
subject: "ce_subject",
|
|
22
|
+
correlationId: "ce_correlationid",
|
|
23
|
+
causationId: "ce_causationid"
|
|
24
|
+
};
|
|
25
|
+
var PROTOCOL_HEADERS = /* @__PURE__ */ new Set([...Object.values(HEADER), CONTENT_TYPE]);
|
|
26
|
+
var RFC3339_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/i;
|
|
27
|
+
var strictUtf8 = new TextDecoder("utf-8", { fatal: true });
|
|
28
|
+
var lenientUtf8 = new TextDecoder("utf-8");
|
|
29
|
+
function encodeEnvelope(envelope) {
|
|
30
|
+
const value = serializePayload(envelope);
|
|
31
|
+
const entries = Object.entries(envelope.headers).map(
|
|
32
|
+
([name, headerValue]) => [escapeUserHeaderName(name), headerValue]
|
|
33
|
+
);
|
|
34
|
+
entries.push(
|
|
35
|
+
[HEADER.specVersion, SPEC_VERSION],
|
|
36
|
+
[HEADER.id, envelope.id],
|
|
37
|
+
[HEADER.type, envelope.type],
|
|
38
|
+
[HEADER.source, envelope.source],
|
|
39
|
+
[HEADER.time, formatTime(envelope)],
|
|
40
|
+
[CONTENT_TYPE, JSON_MEDIA_TYPE]
|
|
41
|
+
);
|
|
42
|
+
if (envelope.key !== null) {
|
|
43
|
+
entries.push([HEADER.subject, envelope.key]);
|
|
44
|
+
}
|
|
45
|
+
if (envelope.correlationId !== null) {
|
|
46
|
+
entries.push([HEADER.correlationId, envelope.correlationId]);
|
|
47
|
+
}
|
|
48
|
+
if (envelope.causationId !== null) {
|
|
49
|
+
entries.push([HEADER.causationId, envelope.causationId]);
|
|
50
|
+
}
|
|
51
|
+
return { key: envelope.key, value, headers: Object.fromEntries(entries) };
|
|
52
|
+
}
|
|
53
|
+
function serializePayload(envelope) {
|
|
54
|
+
let value;
|
|
55
|
+
try {
|
|
56
|
+
value = JSON.stringify(envelope.payload);
|
|
57
|
+
} catch (error) {
|
|
58
|
+
throw new UnencodableEventError(
|
|
59
|
+
`Event ${envelope.id} has a payload that JSON cannot represent: ${error.message}`
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
if (value === void 0) {
|
|
63
|
+
throw new UnencodableEventError(
|
|
64
|
+
`Event ${envelope.id} has a payload that JSON cannot represent (undefined, a function or a symbol)`
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
return value;
|
|
68
|
+
}
|
|
69
|
+
function formatTime(envelope) {
|
|
70
|
+
const year = envelope.occurredAt.getUTCFullYear();
|
|
71
|
+
if (Number.isNaN(year) || year < 0 || year > 9999) {
|
|
72
|
+
throw new UnencodableEventError(
|
|
73
|
+
`Event ${envelope.id} has an occurredAt that is not an RFC 3339 instant (invalid date or year outside 0..9999)`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
return envelope.occurredAt.toISOString();
|
|
77
|
+
}
|
|
78
|
+
function decodeEnvelope(message) {
|
|
79
|
+
const headers = readHeaders(message.headers);
|
|
80
|
+
if (headers.get(HEADER.specVersion) !== SPEC_VERSION) {
|
|
81
|
+
throw invalid(`${HEADER.specVersion} must be '${SPEC_VERSION}'`);
|
|
82
|
+
}
|
|
83
|
+
const type = requireHeader(headers, HEADER.type);
|
|
84
|
+
try {
|
|
85
|
+
assertEventType(type);
|
|
86
|
+
} catch (error) {
|
|
87
|
+
throw invalid(`${HEADER.type} is not a valid event type: ${error.message}`);
|
|
88
|
+
}
|
|
89
|
+
assertJsonContentType(headers.get(CONTENT_TYPE));
|
|
90
|
+
return {
|
|
91
|
+
id: requireHeader(headers, HEADER.id),
|
|
92
|
+
type,
|
|
93
|
+
source: requireHeader(headers, HEADER.source),
|
|
94
|
+
key: resolveKey(headers.get(HEADER.subject), message.key),
|
|
95
|
+
payload: parsePayload(message.value),
|
|
96
|
+
headers: collectUserHeaders(headers),
|
|
97
|
+
correlationId: headers.get(HEADER.correlationId) ?? null,
|
|
98
|
+
causationId: headers.get(HEADER.causationId) ?? null,
|
|
99
|
+
occurredAt: resolveTime(headers.get(HEADER.time), message.timestamp)
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
function invalid(reason) {
|
|
103
|
+
return new InvalidKafkaMessageError(reason);
|
|
104
|
+
}
|
|
105
|
+
function escapeUserHeaderName(name) {
|
|
106
|
+
const lowered = name.toLowerCase();
|
|
107
|
+
const reserved = lowered === CONTENT_TYPE || RESERVED_PREFIXES.some((p) => lowered.startsWith(p));
|
|
108
|
+
return reserved ? `${ESCAPE_PREFIX}${name}` : name;
|
|
109
|
+
}
|
|
110
|
+
function readHeaders(incoming) {
|
|
111
|
+
const headers = /* @__PURE__ */ new Map();
|
|
112
|
+
for (const [name, raw] of Object.entries(incoming ?? {})) {
|
|
113
|
+
if (raw === void 0) {
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (typeof raw !== "string" && !(raw instanceof Uint8Array)) {
|
|
117
|
+
throw invalid(`Header '${name}' is repeated`);
|
|
118
|
+
}
|
|
119
|
+
headers.set(name, decodeText(raw, PROTOCOL_HEADERS.has(name), `Header '${name}'`));
|
|
120
|
+
}
|
|
121
|
+
return headers;
|
|
122
|
+
}
|
|
123
|
+
function decodeText(raw, strict, label) {
|
|
124
|
+
if (typeof raw === "string") {
|
|
125
|
+
return raw;
|
|
126
|
+
}
|
|
127
|
+
try {
|
|
128
|
+
return (strict ? strictUtf8 : lenientUtf8).decode(raw);
|
|
129
|
+
} catch {
|
|
130
|
+
throw invalid(`${label} is not valid UTF-8`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
function requireHeader(headers, name) {
|
|
134
|
+
const value = headers.get(name);
|
|
135
|
+
if (value === void 0 || value.length === 0) {
|
|
136
|
+
throw invalid(`Header '${name}' is missing or empty`);
|
|
137
|
+
}
|
|
138
|
+
return value;
|
|
139
|
+
}
|
|
140
|
+
function assertJsonContentType(contentType) {
|
|
141
|
+
const mediaType = contentType?.split(";")[0]?.trim().toLowerCase();
|
|
142
|
+
if (mediaType !== JSON_MEDIA_TYPE) {
|
|
143
|
+
throw invalid(`${CONTENT_TYPE} must be '${JSON_MEDIA_TYPE}', received '${contentType ?? ""}'`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function resolveTime(text, recordTimestamp) {
|
|
147
|
+
if (text !== void 0 && text.length > 0) {
|
|
148
|
+
return parseTime(text);
|
|
149
|
+
}
|
|
150
|
+
const milliseconds = Number(recordTimestamp);
|
|
151
|
+
const time = new Date(milliseconds);
|
|
152
|
+
if (recordTimestamp === void 0 || recordTimestamp === "" || Number.isNaN(time.getTime())) {
|
|
153
|
+
throw invalid(`${HEADER.time} is missing and the record has no usable timestamp`);
|
|
154
|
+
}
|
|
155
|
+
return time;
|
|
156
|
+
}
|
|
157
|
+
function parseTime(text) {
|
|
158
|
+
const time = new Date(text);
|
|
159
|
+
if (!RFC3339_PATTERN.test(text) || Number.isNaN(time.getTime())) {
|
|
160
|
+
throw invalid(`${HEADER.time} is not an RFC 3339 timestamp: '${text}'`);
|
|
161
|
+
}
|
|
162
|
+
return time;
|
|
163
|
+
}
|
|
164
|
+
function parsePayload(value) {
|
|
165
|
+
if (value === null) {
|
|
166
|
+
throw invalid("Message has no value");
|
|
167
|
+
}
|
|
168
|
+
const text = decodeText(value, true, "Message value");
|
|
169
|
+
try {
|
|
170
|
+
return JSON.parse(text);
|
|
171
|
+
} catch {
|
|
172
|
+
throw invalid("Message value is not valid JSON");
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function resolveKey(subject, recordKey) {
|
|
176
|
+
const keyText = recordKey === null ? null : decodeText(recordKey, true, "Message key");
|
|
177
|
+
if (subject !== void 0 && keyText !== null && subject !== keyText) {
|
|
178
|
+
throw invalid(`${HEADER.subject} does not match the message key`);
|
|
179
|
+
}
|
|
180
|
+
return subject ?? keyText;
|
|
181
|
+
}
|
|
182
|
+
function collectUserHeaders(headers) {
|
|
183
|
+
const entries = [];
|
|
184
|
+
for (const [name, value] of headers) {
|
|
185
|
+
if (!PROTOCOL_HEADERS.has(name)) {
|
|
186
|
+
entries.push([
|
|
187
|
+
name.startsWith(ESCAPE_PREFIX) ? name.slice(ESCAPE_PREFIX.length) : name,
|
|
188
|
+
value
|
|
189
|
+
]);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return Object.fromEntries(entries);
|
|
193
|
+
}
|
|
194
|
+
function toOutgoingHeaders(incoming) {
|
|
195
|
+
const entries = [];
|
|
196
|
+
for (const [name, raw] of Object.entries(incoming ?? {})) {
|
|
197
|
+
if (raw !== void 0) {
|
|
198
|
+
entries.push([
|
|
199
|
+
name,
|
|
200
|
+
Array.isArray(raw) ? raw.map(toBufferOrString) : toBufferOrString(raw)
|
|
201
|
+
]);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return Object.fromEntries(entries);
|
|
205
|
+
}
|
|
206
|
+
function toBufferOrString(value) {
|
|
207
|
+
return typeof value === "string" ? value : Buffer.from(value);
|
|
208
|
+
}
|
|
209
|
+
var PublishTimeoutError = class extends RelayBusError {
|
|
210
|
+
name = "PublishTimeoutError";
|
|
211
|
+
};
|
|
212
|
+
var HandlerFailedError = class extends RelayBusError {
|
|
213
|
+
name = "HandlerFailedError";
|
|
214
|
+
};
|
|
215
|
+
var DEFAULTS = {
|
|
216
|
+
name: "kafka",
|
|
217
|
+
maxInFlightRequests: 1,
|
|
218
|
+
maxAttempts: 5,
|
|
219
|
+
initialBackoff: "200ms",
|
|
220
|
+
maxBackoff: "5s",
|
|
221
|
+
startupTimeout: "30s",
|
|
222
|
+
publishTimeout: "30s",
|
|
223
|
+
heartbeatInterval: "1s",
|
|
224
|
+
restartInitialBackoff: "1s",
|
|
225
|
+
restartMaxBackoff: "30s"
|
|
226
|
+
};
|
|
227
|
+
var KafkaTransport = class {
|
|
228
|
+
name;
|
|
229
|
+
#kafka;
|
|
230
|
+
#groupId;
|
|
231
|
+
#fromBeginning;
|
|
232
|
+
#maxInFlightRequests;
|
|
233
|
+
#producerConfig;
|
|
234
|
+
#consumerConfig;
|
|
235
|
+
#invalidPolicy;
|
|
236
|
+
#logger;
|
|
237
|
+
#settings;
|
|
238
|
+
#sleepers = /* @__PURE__ */ new Set();
|
|
239
|
+
#state = "idle";
|
|
240
|
+
#producer;
|
|
241
|
+
#consumer;
|
|
242
|
+
#groupJoined = false;
|
|
243
|
+
#consuming = false;
|
|
244
|
+
#producerConnected = false;
|
|
245
|
+
#restartLoop;
|
|
246
|
+
#stalledPartitions = /* @__PURE__ */ new Set();
|
|
247
|
+
#startConsuming;
|
|
248
|
+
#joinWaiter;
|
|
249
|
+
constructor(options) {
|
|
250
|
+
this.name = options.name ?? DEFAULTS.name;
|
|
251
|
+
this.#kafka = options.kafka;
|
|
252
|
+
this.#groupId = options.groupId;
|
|
253
|
+
this.#fromBeginning = options.fromBeginning ?? true;
|
|
254
|
+
this.#maxInFlightRequests = options.maxInFlightRequests ?? DEFAULTS.maxInFlightRequests;
|
|
255
|
+
if (!Number.isInteger(this.#maxInFlightRequests) || this.#maxInFlightRequests < 1 || this.#maxInFlightRequests > 5) {
|
|
256
|
+
throw new ConfigurationError(
|
|
257
|
+
`maxInFlightRequests must be an integer from 1 to 5, received ${this.#maxInFlightRequests}`
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
this.#producerConfig = options.producerConfig ?? {};
|
|
261
|
+
this.#consumerConfig = options.consumerConfig ?? {};
|
|
262
|
+
this.#invalidPolicy = options.onInvalidMessage ?? "dead-letter";
|
|
263
|
+
this.#logger = options.logger ?? console;
|
|
264
|
+
this.#settings = resolveSettings(options);
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* True while the consumer is joined to its group, not between a crash or
|
|
268
|
+
* rebalance and the next join, and no partition is stalled on a record whose
|
|
269
|
+
* handler keeps failing. Use it for readiness so a consumer that silently
|
|
270
|
+
* stopped is noticed.
|
|
271
|
+
*/
|
|
272
|
+
isConsuming() {
|
|
273
|
+
return this.#state === "subscribed" && this.#groupJoined && this.#stalledPartitions.size === 0;
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* True once the producer is connected and, if the transport subscribed, the
|
|
277
|
+
* consumer is consuming. False before `connect`, after `close` and while a
|
|
278
|
+
* dead consumer is being restarted.
|
|
279
|
+
*/
|
|
280
|
+
isReady() {
|
|
281
|
+
if (this.#state === "closed" || !this.#producerConnected) {
|
|
282
|
+
return false;
|
|
283
|
+
}
|
|
284
|
+
return this.#state === "idle" || this.isConsuming();
|
|
285
|
+
}
|
|
286
|
+
/** Connects the producer. Called by the bus on start; `publish` also connects on demand. */
|
|
287
|
+
async connect() {
|
|
288
|
+
await this.#connectedProducer();
|
|
289
|
+
}
|
|
290
|
+
async publish(envelope, destination) {
|
|
291
|
+
const producer = await this.#connectedProducer();
|
|
292
|
+
const { key, value, headers } = encodeEnvelope(envelope);
|
|
293
|
+
await this.#send(producer, {
|
|
294
|
+
topic: destination,
|
|
295
|
+
acks: -1,
|
|
296
|
+
messages: [{ key, value, headers }]
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Starts the consumer and resolves once it joined its group, so a start-up
|
|
301
|
+
* that cannot consume fails loudly instead of looking healthy.
|
|
302
|
+
*
|
|
303
|
+
* @throws ConfigurationError when `groupId` is missing, `destinations` is empty
|
|
304
|
+
* or the transport already subscribed (create one transport per subscription).
|
|
305
|
+
*/
|
|
306
|
+
async subscribe(destinations, onMessage) {
|
|
307
|
+
this.#assertCanSubscribe(destinations);
|
|
308
|
+
this.#state = "subscribed";
|
|
309
|
+
const consumer = this.#kafka.consumer({ ...this.#consumerConfig, groupId: this.#groupId });
|
|
310
|
+
this.#consumer = consumer;
|
|
311
|
+
this.#observeConsumer(consumer);
|
|
312
|
+
const start = () => this.#startConsumer(consumer, [...destinations], onMessage);
|
|
313
|
+
this.#startConsuming = start;
|
|
314
|
+
try {
|
|
315
|
+
await start();
|
|
316
|
+
} catch (error) {
|
|
317
|
+
await consumer.disconnect().catch(() => void 0);
|
|
318
|
+
this.#groupJoined = false;
|
|
319
|
+
this.#consumer = void 0;
|
|
320
|
+
this.#state = "idle";
|
|
321
|
+
throw error;
|
|
322
|
+
}
|
|
323
|
+
this.#consuming = true;
|
|
324
|
+
this.#logger.log(
|
|
325
|
+
`[KafkaTransport](subscribe) Consuming | transport=${this.name} group=${this.#groupId} topics=${destinations.join(",")}`
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
async #startConsumer(consumer, destinations, onMessage) {
|
|
329
|
+
await consumer.connect();
|
|
330
|
+
await consumer.subscribe({ topics: destinations, fromBeginning: this.#fromBeginning });
|
|
331
|
+
const joined = this.#waitForGroupJoin(consumer);
|
|
332
|
+
try {
|
|
333
|
+
await consumer.run({ eachMessage: (payload) => this.#handleMessage(payload, onMessage) });
|
|
334
|
+
await joined.promise;
|
|
335
|
+
} finally {
|
|
336
|
+
joined.dispose();
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
/** Stops consuming and disconnects the consumer and the producer. Safe to call twice. */
|
|
340
|
+
async close() {
|
|
341
|
+
if (this.#state === "closed") {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
this.#state = "closed";
|
|
345
|
+
const pendingProducer = this.#producer;
|
|
346
|
+
this.#groupJoined = false;
|
|
347
|
+
this.#producerConnected = false;
|
|
348
|
+
for (const wake of this.#sleepers) {
|
|
349
|
+
wake();
|
|
350
|
+
}
|
|
351
|
+
this.#joinWaiter?.reject(new RelayBusError(`Transport '${this.name}' is closed`));
|
|
352
|
+
await this.#restartLoop;
|
|
353
|
+
const producer = await pendingProducer?.catch(() => void 0);
|
|
354
|
+
const results = await Promise.allSettled([
|
|
355
|
+
this.#consumer?.disconnect(),
|
|
356
|
+
producer?.disconnect()
|
|
357
|
+
]);
|
|
358
|
+
const failure = results.find((result) => result.status === "rejected");
|
|
359
|
+
if (failure !== void 0) {
|
|
360
|
+
throw failure.reason;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
#assertCanSubscribe(destinations) {
|
|
364
|
+
if (this.#state === "closed") {
|
|
365
|
+
throw new RelayBusError(`Transport '${this.name}' is closed`);
|
|
366
|
+
}
|
|
367
|
+
if (this.#state === "subscribed") {
|
|
368
|
+
throw new ConfigurationError(
|
|
369
|
+
`Transport '${this.name}' already subscribed; list every topic in one subscription or use one KafkaTransport per consumer group`
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
if (this.#groupId === void 0 || this.#groupId.length === 0) {
|
|
373
|
+
throw new ConfigurationError(`Transport '${this.name}' needs a groupId to subscribe`);
|
|
374
|
+
}
|
|
375
|
+
if (destinations.length === 0) {
|
|
376
|
+
throw new ConfigurationError(`Transport '${this.name}' subscribed to no topics`);
|
|
377
|
+
}
|
|
378
|
+
for (const topic of destinations) {
|
|
379
|
+
const deadLetterTopic = this.#deadLetterTopicFor(topic);
|
|
380
|
+
if (deadLetterTopic !== void 0 && destinations.includes(deadLetterTopic)) {
|
|
381
|
+
throw new ConfigurationError(
|
|
382
|
+
`Dead letter topic '${deadLetterTopic}' is also consumed, which would loop`
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
/** The dead-letter topic for records of `sourceTopic`, or undefined when the policy does not use one. */
|
|
388
|
+
#deadLetterTopicFor(sourceTopic) {
|
|
389
|
+
const policy = this.#invalidPolicy;
|
|
390
|
+
if (policy === "dead-letter") {
|
|
391
|
+
return `${sourceTopic}.dead`;
|
|
392
|
+
}
|
|
393
|
+
if (typeof policy === "object") {
|
|
394
|
+
const { deadLetterTopic } = policy;
|
|
395
|
+
return typeof deadLetterTopic === "function" ? deadLetterTopic(sourceTopic) : deadLetterTopic;
|
|
396
|
+
}
|
|
397
|
+
return void 0;
|
|
398
|
+
}
|
|
399
|
+
/** Sends with the publish timeout; the late outcome of a timed-out send is ignored. */
|
|
400
|
+
async #send(producer, record) {
|
|
401
|
+
let timer;
|
|
402
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
403
|
+
timer = setTimeout(() => {
|
|
404
|
+
reject(
|
|
405
|
+
new PublishTimeoutError(
|
|
406
|
+
`Transport '${this.name}' got no broker ack for topic '${record.topic}' within ${this.#settings.publishTimeoutMs}ms`
|
|
407
|
+
)
|
|
408
|
+
);
|
|
409
|
+
}, this.#settings.publishTimeoutMs);
|
|
410
|
+
});
|
|
411
|
+
const sending = producer.send(record);
|
|
412
|
+
sending.catch(() => void 0);
|
|
413
|
+
try {
|
|
414
|
+
await Promise.race([sending, timeout]);
|
|
415
|
+
} finally {
|
|
416
|
+
clearTimeout(timer);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
#connectedProducer() {
|
|
420
|
+
if (this.#state === "closed") {
|
|
421
|
+
return Promise.reject(new RelayBusError(`Transport '${this.name}' is closed`));
|
|
422
|
+
}
|
|
423
|
+
this.#producer ??= this.#openProducer();
|
|
424
|
+
return this.#producer;
|
|
425
|
+
}
|
|
426
|
+
async #openProducer() {
|
|
427
|
+
const producer = this.#kafka.producer({
|
|
428
|
+
...this.#producerConfig,
|
|
429
|
+
idempotent: true,
|
|
430
|
+
maxInFlightRequests: this.#maxInFlightRequests
|
|
431
|
+
});
|
|
432
|
+
try {
|
|
433
|
+
await producer.connect();
|
|
434
|
+
} catch (error) {
|
|
435
|
+
this.#producer = void 0;
|
|
436
|
+
throw error;
|
|
437
|
+
}
|
|
438
|
+
this.#producerConnected = this.#state !== "closed";
|
|
439
|
+
return producer;
|
|
440
|
+
}
|
|
441
|
+
async #handleMessage(payload, onMessage) {
|
|
442
|
+
let envelope;
|
|
443
|
+
try {
|
|
444
|
+
envelope = decodeEnvelope(payload.message);
|
|
445
|
+
} catch (error) {
|
|
446
|
+
if (!(error instanceof InvalidKafkaMessageError)) {
|
|
447
|
+
throw toHandlerFailure(error);
|
|
448
|
+
}
|
|
449
|
+
await this.#withRetries(
|
|
450
|
+
payload,
|
|
451
|
+
"handleInvalid",
|
|
452
|
+
() => this.#handleInvalid(payload, error.message)
|
|
453
|
+
);
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
await this.#withRetries(payload, "deliver", async () => {
|
|
457
|
+
try {
|
|
458
|
+
await onMessage(envelope);
|
|
459
|
+
} catch (error) {
|
|
460
|
+
if (!(error instanceof NonRetryableError)) {
|
|
461
|
+
throw error;
|
|
462
|
+
}
|
|
463
|
+
await this.#handleInvalid(payload, error.message);
|
|
464
|
+
}
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
async #handleInvalid(payload, reason) {
|
|
468
|
+
const { topic, partition, message } = payload;
|
|
469
|
+
const location = `topic=${topic} partition=${partition} offset=${message.offset}`;
|
|
470
|
+
const policy = this.#invalidPolicy;
|
|
471
|
+
const deadLetterTopic = this.#deadLetterTopicFor(topic);
|
|
472
|
+
if (policy === "skip") {
|
|
473
|
+
this.#logger.error(
|
|
474
|
+
`[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason}`
|
|
475
|
+
);
|
|
476
|
+
} else if (typeof policy === "function") {
|
|
477
|
+
await policy({
|
|
478
|
+
topic,
|
|
479
|
+
partition,
|
|
480
|
+
offset: message.offset,
|
|
481
|
+
reason,
|
|
482
|
+
key: message.key,
|
|
483
|
+
value: message.value
|
|
484
|
+
});
|
|
485
|
+
} else {
|
|
486
|
+
const producer = await this.#connectedProducer();
|
|
487
|
+
await this.#send(producer, {
|
|
488
|
+
topic: deadLetterTopic,
|
|
489
|
+
acks: -1,
|
|
490
|
+
messages: [
|
|
491
|
+
{
|
|
492
|
+
key: message.key === null ? null : Buffer.from(message.key),
|
|
493
|
+
value: message.value === null ? null : Buffer.from(message.value),
|
|
494
|
+
headers: {
|
|
495
|
+
...toOutgoingHeaders(message.headers),
|
|
496
|
+
"relaybus-invalid-reason": reason,
|
|
497
|
+
"relaybus-source-topic": topic,
|
|
498
|
+
"relaybus-source-partition": String(partition),
|
|
499
|
+
"relaybus-source-offset": message.offset
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
]
|
|
503
|
+
});
|
|
504
|
+
this.#logger.error(
|
|
505
|
+
`[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason}`
|
|
506
|
+
);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* Runs `operation` up to `maxAttempts` times with exponential backoff. The
|
|
511
|
+
* last error is rethrown so the client leaves the offset uncommitted and
|
|
512
|
+
* redelivers the record; heartbeats between attempts keep the session alive.
|
|
513
|
+
*/
|
|
514
|
+
async #withRetries(payload, method, operation) {
|
|
515
|
+
const { maxAttempts, initialBackoffMs, maxBackoffMs } = this.#settings;
|
|
516
|
+
const location = `topic=${payload.topic} partition=${payload.partition} offset=${payload.message.offset}`;
|
|
517
|
+
const partitionKey = `${payload.topic}:${payload.partition}`;
|
|
518
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
519
|
+
try {
|
|
520
|
+
await operation();
|
|
521
|
+
this.#stalledPartitions.delete(partitionKey);
|
|
522
|
+
return;
|
|
523
|
+
} catch (error) {
|
|
524
|
+
const reason = toErrorInfo(error).message;
|
|
525
|
+
if (attempt >= maxAttempts || this.#state === "closed") {
|
|
526
|
+
this.#stalledPartitions.add(partitionKey);
|
|
527
|
+
this.#logger.error(
|
|
528
|
+
`[KafkaTransport](${method}) Giving up for now, the record stays uncommitted and will be redelivered | ${location} attempts=${attempt} error=${reason}`
|
|
529
|
+
);
|
|
530
|
+
throw toHandlerFailure(error);
|
|
531
|
+
}
|
|
532
|
+
this.#logger.warn(
|
|
533
|
+
`[KafkaTransport](${method}) Attempt failed, retrying | ${location} attempt=${attempt} error=${reason}`
|
|
534
|
+
);
|
|
535
|
+
await this.#sleepWithHeartbeats(
|
|
536
|
+
Math.min(initialBackoffMs * 2 ** (attempt - 1), maxBackoffMs),
|
|
537
|
+
payload
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
/** Sleeps in slices and heartbeats after each, so a long backoff never outlives the session timeout. */
|
|
543
|
+
async #sleepWithHeartbeats(milliseconds, payload) {
|
|
544
|
+
let remaining = milliseconds;
|
|
545
|
+
do {
|
|
546
|
+
const slice = Math.min(remaining, this.#settings.heartbeatIntervalMs);
|
|
547
|
+
await this.#sleep(slice);
|
|
548
|
+
await payload.heartbeat();
|
|
549
|
+
remaining -= slice;
|
|
550
|
+
} while (remaining > 0 && this.#state !== "closed");
|
|
551
|
+
}
|
|
552
|
+
#sleep(milliseconds) {
|
|
553
|
+
return new Promise((resolve) => {
|
|
554
|
+
const wake = () => {
|
|
555
|
+
clearTimeout(timer);
|
|
556
|
+
this.#sleepers.delete(wake);
|
|
557
|
+
resolve();
|
|
558
|
+
};
|
|
559
|
+
const timer = setTimeout(wake, milliseconds);
|
|
560
|
+
this.#sleepers.add(wake);
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
/** Whether the client exposes the instrumentation needed to follow the group membership. */
|
|
564
|
+
#tracksMembership(consumer) {
|
|
565
|
+
return consumer.on !== void 0 && consumer.events?.GROUP_JOIN !== void 0;
|
|
566
|
+
}
|
|
567
|
+
#observeConsumer(consumer) {
|
|
568
|
+
if (!this.#tracksMembership(consumer)) {
|
|
569
|
+
this.#groupJoined = true;
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
const names = consumer.events;
|
|
573
|
+
const listen = (name, listener) => {
|
|
574
|
+
if (name !== void 0) {
|
|
575
|
+
consumer.on(name, listener);
|
|
576
|
+
}
|
|
577
|
+
};
|
|
578
|
+
listen(names.GROUP_JOIN, () => {
|
|
579
|
+
this.#groupJoined = true;
|
|
580
|
+
this.#joinWaiter?.resolve();
|
|
581
|
+
this.#logger.log(`[KafkaTransport](observeConsumer) Joined group | group=${this.#groupId}`);
|
|
582
|
+
});
|
|
583
|
+
listen(names.REBALANCING, () => {
|
|
584
|
+
this.#groupJoined = false;
|
|
585
|
+
this.#stalledPartitions.clear();
|
|
586
|
+
this.#logger.warn(`[KafkaTransport](observeConsumer) Rebalancing | group=${this.#groupId}`);
|
|
587
|
+
});
|
|
588
|
+
listen(names.CRASH, (event) => this.#handleCrash(event));
|
|
589
|
+
listen(names.STOP, () => {
|
|
590
|
+
this.#groupJoined = false;
|
|
591
|
+
});
|
|
592
|
+
listen(names.DISCONNECT, () => {
|
|
593
|
+
this.#groupJoined = false;
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
#handleCrash(event) {
|
|
597
|
+
const { error, restart } = event.payload ?? {};
|
|
598
|
+
this.#groupJoined = false;
|
|
599
|
+
this.#logger.error(
|
|
600
|
+
`[KafkaTransport](handleCrash) Consumer crashed | group=${this.#groupId} restart=${String(restart)} error=${toErrorInfo(error).message}`
|
|
601
|
+
);
|
|
602
|
+
this.#joinWaiter?.reject(error);
|
|
603
|
+
if (restart === false && this.#consuming && this.#restartLoop === void 0) {
|
|
604
|
+
this.#restartLoop = this.#restartConsumer().finally(() => {
|
|
605
|
+
this.#restartLoop = void 0;
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* The client gave up on the consumer (`restart=false`): reconnect it and run
|
|
611
|
+
* again, with bounded backoff, until it joined or the transport closed.
|
|
612
|
+
*/
|
|
613
|
+
async #restartConsumer() {
|
|
614
|
+
const { restartInitialBackoffMs, restartMaxBackoffMs } = this.#settings;
|
|
615
|
+
const consumer = this.#consumer;
|
|
616
|
+
for (let attempt = 1; this.#state === "subscribed"; attempt += 1) {
|
|
617
|
+
await this.#sleep(
|
|
618
|
+
Math.min(restartInitialBackoffMs * 2 ** (attempt - 1), restartMaxBackoffMs)
|
|
619
|
+
);
|
|
620
|
+
if (this.#state !== "subscribed") {
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
await consumer.disconnect().catch(() => void 0);
|
|
624
|
+
try {
|
|
625
|
+
await this.#startConsuming();
|
|
626
|
+
this.#logger.log(
|
|
627
|
+
`[KafkaTransport](restartConsumer) Consumer restarted | group=${this.#groupId} attempt=${attempt}`
|
|
628
|
+
);
|
|
629
|
+
return;
|
|
630
|
+
} catch (error) {
|
|
631
|
+
this.#logger.error(
|
|
632
|
+
`[KafkaTransport](restartConsumer) Restart failed, will retry | group=${this.#groupId} attempt=${attempt} error=${toErrorInfo(error).message}`
|
|
633
|
+
);
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
#waitForGroupJoin(consumer) {
|
|
638
|
+
if (!this.#tracksMembership(consumer)) {
|
|
639
|
+
return { promise: Promise.resolve(), dispose: () => void 0 };
|
|
640
|
+
}
|
|
641
|
+
let timer;
|
|
642
|
+
const promise = new Promise((resolve, reject) => {
|
|
643
|
+
this.#joinWaiter = { resolve, reject };
|
|
644
|
+
timer = setTimeout(() => {
|
|
645
|
+
reject(
|
|
646
|
+
new RelayBusError(
|
|
647
|
+
`Consumer of transport '${this.name}' did not join group '${this.#groupId}' within ${this.#settings.startupTimeoutMs}ms`
|
|
648
|
+
)
|
|
649
|
+
);
|
|
650
|
+
}, this.#settings.startupTimeoutMs);
|
|
651
|
+
});
|
|
652
|
+
promise.catch(() => void 0);
|
|
653
|
+
return {
|
|
654
|
+
promise,
|
|
655
|
+
dispose: () => {
|
|
656
|
+
clearTimeout(timer);
|
|
657
|
+
this.#joinWaiter = void 0;
|
|
658
|
+
}
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
};
|
|
662
|
+
function resolveSettings(options) {
|
|
663
|
+
const maxAttempts = options.processing?.maxAttempts ?? DEFAULTS.maxAttempts;
|
|
664
|
+
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
|
665
|
+
throw new ConfigurationError(
|
|
666
|
+
`processing.maxAttempts must be a positive integer, received ${maxAttempts}`
|
|
667
|
+
);
|
|
668
|
+
}
|
|
669
|
+
return {
|
|
670
|
+
maxAttempts,
|
|
671
|
+
initialBackoffMs: toMilliseconds(
|
|
672
|
+
options.processing?.initialBackoff ?? DEFAULTS.initialBackoff,
|
|
673
|
+
"processing.initialBackoff"
|
|
674
|
+
),
|
|
675
|
+
maxBackoffMs: toMilliseconds(
|
|
676
|
+
options.processing?.maxBackoff ?? DEFAULTS.maxBackoff,
|
|
677
|
+
"processing.maxBackoff"
|
|
678
|
+
),
|
|
679
|
+
startupTimeoutMs: toMilliseconds(
|
|
680
|
+
options.startupTimeout ?? DEFAULTS.startupTimeout,
|
|
681
|
+
"startupTimeout"
|
|
682
|
+
),
|
|
683
|
+
publishTimeoutMs: toMilliseconds(
|
|
684
|
+
options.publishTimeout ?? DEFAULTS.publishTimeout,
|
|
685
|
+
"publishTimeout"
|
|
686
|
+
),
|
|
687
|
+
heartbeatIntervalMs: Math.max(
|
|
688
|
+
1,
|
|
689
|
+
toMilliseconds(
|
|
690
|
+
options.processing?.heartbeatInterval ?? DEFAULTS.heartbeatInterval,
|
|
691
|
+
"processing.heartbeatInterval"
|
|
692
|
+
)
|
|
693
|
+
),
|
|
694
|
+
restartInitialBackoffMs: toMilliseconds(
|
|
695
|
+
options.restart?.initialBackoff ?? DEFAULTS.restartInitialBackoff,
|
|
696
|
+
"restart.initialBackoff"
|
|
697
|
+
),
|
|
698
|
+
restartMaxBackoffMs: toMilliseconds(
|
|
699
|
+
options.restart?.maxBackoff ?? DEFAULTS.restartMaxBackoff,
|
|
700
|
+
"restart.maxBackoff"
|
|
701
|
+
)
|
|
702
|
+
};
|
|
703
|
+
}
|
|
704
|
+
function toHandlerFailure(error) {
|
|
705
|
+
const { name, message } = toErrorInfo(error);
|
|
706
|
+
return new HandlerFailedError(`${name}: ${message}`);
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
export { HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, UnencodableEventError, decodeEnvelope, encodeEnvelope };
|
|
710
|
+
//# sourceMappingURL=index.js.map
|
|
711
|
+
//# sourceMappingURL=index.js.map
|