@relaybus/kafka 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -48
- package/dist/index.cjs +267 -169
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +123 -62
- package/dist/index.d.ts +123 -62
- package/dist/index.js +256 -168
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,14 +1,36 @@
|
|
|
1
|
-
import { RelayBusError, NonRetryableError, assertEventType, ConfigurationError, toErrorInfo, toMilliseconds } from '@relaybus/core';
|
|
1
|
+
import { RelayBusError, NonRetryableError, assertEventType, ConfigurationError, systemClock, toErrorInfo, toMilliseconds } from '@relaybus/core';
|
|
2
|
+
import { createCloudEventReaders, mediaTypeOf, isJsonObject, createJsonEnvelopeFormat, schedule, assertPositiveInteger, writeStructuredCloudEvent } from '@relaybus/core/internal';
|
|
3
|
+
export { fromHeader, fromPayloadField } from '@relaybus/core/internal';
|
|
2
4
|
|
|
3
|
-
// src/
|
|
5
|
+
// src/errors.ts
|
|
4
6
|
var InvalidKafkaMessageError = class extends RelayBusError {
|
|
7
|
+
name = "InvalidKafkaMessageError";
|
|
8
|
+
code = "RELAYBUS_KAFKA_INVALID_MESSAGE";
|
|
5
9
|
};
|
|
6
10
|
var UnencodableEventError = class extends NonRetryableError {
|
|
7
11
|
name = "UnencodableEventError";
|
|
12
|
+
code = "RELAYBUS_KAFKA_UNENCODABLE_EVENT";
|
|
13
|
+
};
|
|
14
|
+
var PublishTimeoutError = class extends RelayBusError {
|
|
15
|
+
name = "PublishTimeoutError";
|
|
16
|
+
code = "RELAYBUS_KAFKA_PUBLISH_TIMEOUT";
|
|
17
|
+
};
|
|
18
|
+
var TransportClosedError = class extends RelayBusError {
|
|
19
|
+
name = "TransportClosedError";
|
|
20
|
+
code = "RELAYBUS_KAFKA_TRANSPORT_CLOSED";
|
|
21
|
+
};
|
|
22
|
+
var GroupJoinTimeoutError = class extends RelayBusError {
|
|
23
|
+
name = "GroupJoinTimeoutError";
|
|
24
|
+
code = "RELAYBUS_KAFKA_GROUP_JOIN_TIMEOUT";
|
|
25
|
+
};
|
|
26
|
+
var HandlerFailedError = class extends RelayBusError {
|
|
27
|
+
name = "HandlerFailedError";
|
|
28
|
+
code = "RELAYBUS_KAFKA_HANDLER_FAILED";
|
|
8
29
|
};
|
|
9
30
|
var SPEC_VERSION = "1.0";
|
|
10
31
|
var CONTENT_TYPE = "content-type";
|
|
11
32
|
var JSON_MEDIA_TYPE = "application/json";
|
|
33
|
+
var STRUCTURED_MEDIA_TYPE = "application/cloudevents+json";
|
|
12
34
|
var CLOUDEVENTS_PREFIX = "ce_";
|
|
13
35
|
var ESCAPE_PREFIX = "x-relaybus-h-";
|
|
14
36
|
var RESERVED_PREFIXES = [CLOUDEVENTS_PREFIX, "x-relaybus-"];
|
|
@@ -23,6 +45,7 @@ var HEADER = {
|
|
|
23
45
|
causationId: "ce_causationid"
|
|
24
46
|
};
|
|
25
47
|
var PROTOCOL_HEADERS = /* @__PURE__ */ new Set([...Object.values(HEADER), CONTENT_TYPE]);
|
|
48
|
+
var MAXIMUM_RFC3339_YEAR = 9999;
|
|
26
49
|
var RFC3339_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/i;
|
|
27
50
|
var strictUtf8 = new TextDecoder("utf-8", { fatal: true });
|
|
28
51
|
var lenientUtf8 = new TextDecoder("utf-8");
|
|
@@ -50,25 +73,39 @@ function encodeEnvelope(envelope) {
|
|
|
50
73
|
}
|
|
51
74
|
return { key: envelope.key, value, headers: Object.fromEntries(entries) };
|
|
52
75
|
}
|
|
76
|
+
function encodeStructuredEnvelope(envelope) {
|
|
77
|
+
const value = writeStructuredCloudEvent(envelope, {
|
|
78
|
+
time: formatTime(envelope),
|
|
79
|
+
data: serializePayload(envelope)
|
|
80
|
+
});
|
|
81
|
+
const entries = Object.entries(envelope.headers).map(
|
|
82
|
+
([name, headerValue]) => [escapeUserHeaderName(name), headerValue]
|
|
83
|
+
);
|
|
84
|
+
entries.push([CONTENT_TYPE, STRUCTURED_MEDIA_TYPE]);
|
|
85
|
+
return { key: envelope.key, value, headers: Object.fromEntries(entries) };
|
|
86
|
+
}
|
|
53
87
|
function serializePayload(envelope) {
|
|
54
|
-
|
|
88
|
+
return serializeJson(envelope, envelope.payload);
|
|
89
|
+
}
|
|
90
|
+
function serializeJson(envelope, value) {
|
|
91
|
+
let text;
|
|
55
92
|
try {
|
|
56
|
-
|
|
93
|
+
text = JSON.stringify(value);
|
|
57
94
|
} catch (error) {
|
|
58
95
|
throw new UnencodableEventError(
|
|
59
96
|
`Event ${envelope.id} has a payload that JSON cannot represent: ${error.message}`
|
|
60
97
|
);
|
|
61
98
|
}
|
|
62
|
-
if (
|
|
99
|
+
if (text === void 0) {
|
|
63
100
|
throw new UnencodableEventError(
|
|
64
101
|
`Event ${envelope.id} has a payload that JSON cannot represent (undefined, a function or a symbol)`
|
|
65
102
|
);
|
|
66
103
|
}
|
|
67
|
-
return
|
|
104
|
+
return text;
|
|
68
105
|
}
|
|
69
106
|
function formatTime(envelope) {
|
|
70
107
|
const year = envelope.occurredAt.getUTCFullYear();
|
|
71
|
-
if (Number.isNaN(year) || year < 0 || year >
|
|
108
|
+
if (Number.isNaN(year) || year < 0 || year > MAXIMUM_RFC3339_YEAR) {
|
|
72
109
|
throw new UnencodableEventError(
|
|
73
110
|
`Event ${envelope.id} has an occurredAt that is not an RFC 3339 instant (invalid date or year outside 0..9999)`
|
|
74
111
|
);
|
|
@@ -116,11 +153,14 @@ function readHeaders(incoming) {
|
|
|
116
153
|
if (typeof raw !== "string" && !(raw instanceof Uint8Array)) {
|
|
117
154
|
throw invalid(`Header '${name}' is repeated`);
|
|
118
155
|
}
|
|
119
|
-
headers.set(
|
|
156
|
+
headers.set(
|
|
157
|
+
name,
|
|
158
|
+
decodeText(raw, { strict: PROTOCOL_HEADERS.has(name), label: `Header '${name}'` })
|
|
159
|
+
);
|
|
120
160
|
}
|
|
121
161
|
return headers;
|
|
122
162
|
}
|
|
123
|
-
function decodeText(raw, strict, label) {
|
|
163
|
+
function decodeText(raw, { strict, label }) {
|
|
124
164
|
if (typeof raw === "string") {
|
|
125
165
|
return raw;
|
|
126
166
|
}
|
|
@@ -143,19 +183,19 @@ function assertJsonContentType(contentType) {
|
|
|
143
183
|
throw invalid(`${CONTENT_TYPE} must be '${JSON_MEDIA_TYPE}', received '${contentType ?? ""}'`);
|
|
144
184
|
}
|
|
145
185
|
}
|
|
146
|
-
function resolveTime(text,
|
|
186
|
+
function resolveTime(text, recordTimestamp2) {
|
|
147
187
|
if (text !== void 0 && text.length > 0) {
|
|
148
188
|
return parseTime(text);
|
|
149
189
|
}
|
|
150
|
-
const time = readRecordTimestamp(
|
|
190
|
+
const time = readRecordTimestamp(recordTimestamp2);
|
|
151
191
|
if (time === void 0) {
|
|
152
192
|
throw invalid(`${HEADER.time} is missing and the record has no usable timestamp`);
|
|
153
193
|
}
|
|
154
194
|
return time;
|
|
155
195
|
}
|
|
156
|
-
function readRecordTimestamp(
|
|
157
|
-
const time = new Date(Number(
|
|
158
|
-
return
|
|
196
|
+
function readRecordTimestamp(recordTimestamp2) {
|
|
197
|
+
const time = new Date(Number(recordTimestamp2));
|
|
198
|
+
return recordTimestamp2 === void 0 || recordTimestamp2 === "" || Number.isNaN(time.getTime()) ? void 0 : time;
|
|
159
199
|
}
|
|
160
200
|
function parseTime(text, label = HEADER.time) {
|
|
161
201
|
const time = new Date(text);
|
|
@@ -168,7 +208,7 @@ function parsePayload(value) {
|
|
|
168
208
|
if (value === null) {
|
|
169
209
|
throw invalid("Message has no value");
|
|
170
210
|
}
|
|
171
|
-
const text = decodeText(value, true, "Message value");
|
|
211
|
+
const text = decodeText(value, { strict: true, label: "Message value" });
|
|
172
212
|
try {
|
|
173
213
|
return JSON.parse(text);
|
|
174
214
|
} catch {
|
|
@@ -183,7 +223,7 @@ function resolveKey(subject, recordKey) {
|
|
|
183
223
|
return subject ?? keyText;
|
|
184
224
|
}
|
|
185
225
|
function readRecordKey(recordKey) {
|
|
186
|
-
return recordKey === null ? null : decodeText(recordKey, true, "Message key");
|
|
226
|
+
return recordKey === null ? null : decodeText(recordKey, { strict: true, label: "Message key" });
|
|
187
227
|
}
|
|
188
228
|
function collectUserHeaders(headers) {
|
|
189
229
|
const entries = [];
|
|
@@ -212,7 +252,8 @@ function toOutgoingHeaders(incoming) {
|
|
|
212
252
|
function toBufferOrString(value) {
|
|
213
253
|
return typeof value === "string" ? value : Buffer.from(value);
|
|
214
254
|
}
|
|
215
|
-
var
|
|
255
|
+
var { optionalAttribute, requiredAttribute, assertValidType, assertJsonDataContentType } = createCloudEventReaders(invalid);
|
|
256
|
+
var STRUCTURED_MEDIA_TYPE2 = "application/cloudevents+json";
|
|
216
257
|
var exactUtf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
217
258
|
var SPEC_VERSION2 = "1.0";
|
|
218
259
|
var SPEC_VERSION_HEADER = "ce_specversion";
|
|
@@ -232,7 +273,7 @@ var STRUCTURED_ATTRIBUTES = /* @__PURE__ */ new Set([
|
|
|
232
273
|
]);
|
|
233
274
|
function decodeCloudEvent(record) {
|
|
234
275
|
const headers = readHeaders(record.headers);
|
|
235
|
-
if (mediaTypeOf(headers.get(CONTENT_TYPE)) !==
|
|
276
|
+
if (mediaTypeOf(headers.get(CONTENT_TYPE)) !== STRUCTURED_MEDIA_TYPE2) {
|
|
236
277
|
return decodeEnvelope(record);
|
|
237
278
|
}
|
|
238
279
|
const body = parsePayload(record.value);
|
|
@@ -267,7 +308,7 @@ function plainJsonDecoder(options) {
|
|
|
267
308
|
}
|
|
268
309
|
return (record) => {
|
|
269
310
|
const headers = readHeadersLeniently(record.headers);
|
|
270
|
-
if (record.headers?.[SPEC_VERSION_HEADER] !== void 0 || mediaTypeOf(headers[CONTENT_TYPE]) ===
|
|
311
|
+
if (record.headers?.[SPEC_VERSION_HEADER] !== void 0 || mediaTypeOf(headers[CONTENT_TYPE]) === STRUCTURED_MEDIA_TYPE2) {
|
|
271
312
|
return decodeCloudEvent(record);
|
|
272
313
|
}
|
|
273
314
|
const plain = {
|
|
@@ -279,21 +320,21 @@ function plainJsonDecoder(options) {
|
|
|
279
320
|
payload: parsePayload(record.value),
|
|
280
321
|
timestamp: record.timestamp
|
|
281
322
|
};
|
|
282
|
-
const type = resolveText(options.type, plain, "type");
|
|
323
|
+
const type = resolveText(options.type, { record: plain, name: "type" });
|
|
283
324
|
if (type === void 0) {
|
|
284
325
|
throw invalid("Record has no event type: the type option found none");
|
|
285
326
|
}
|
|
286
327
|
assertValidType(type, "The event type");
|
|
287
328
|
return {
|
|
288
|
-
id: resolveText(options.id, plain, "id") ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
|
|
329
|
+
id: resolveText(options.id, { record: plain, name: "id" }) ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
|
|
289
330
|
type,
|
|
290
|
-
source: resolveText(options.source, plain, "source") ?? record.topic,
|
|
291
|
-
key: options.key === void 0 ? plain.key : resolveText(options.key, plain, "key") ?? null,
|
|
331
|
+
source: resolveText(options.source, { record: plain, name: "source" }) ?? record.topic,
|
|
332
|
+
key: options.key === void 0 ? plain.key : resolveText(options.key, { record: plain, name: "key" }) ?? null,
|
|
292
333
|
payload: plain.payload,
|
|
293
334
|
headers,
|
|
294
335
|
correlationId: null,
|
|
295
336
|
causationId: null,
|
|
296
|
-
occurredAt: resolveOccurredAt(options.occurredAt, plain, record)
|
|
337
|
+
occurredAt: resolveOccurredAt(options.occurredAt, { plain, record })
|
|
297
338
|
};
|
|
298
339
|
};
|
|
299
340
|
}
|
|
@@ -318,39 +359,6 @@ function completeDecodedEvent(decoded, record) {
|
|
|
318
359
|
occurredAt: event.occurredAt ?? requireRecordTimestamp(record)
|
|
319
360
|
};
|
|
320
361
|
}
|
|
321
|
-
function mediaTypeOf(contentType) {
|
|
322
|
-
return contentType?.split(";")[0]?.trim().toLowerCase();
|
|
323
|
-
}
|
|
324
|
-
function isJsonObject(value) {
|
|
325
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
326
|
-
}
|
|
327
|
-
function optionalAttribute(body, name) {
|
|
328
|
-
const value = body[name];
|
|
329
|
-
if (value !== void 0 && typeof value !== "string") {
|
|
330
|
-
throw invalid(`Attribute '${name}' must be a string`);
|
|
331
|
-
}
|
|
332
|
-
return value;
|
|
333
|
-
}
|
|
334
|
-
function requiredAttribute(body, name) {
|
|
335
|
-
const value = optionalAttribute(body, name);
|
|
336
|
-
if (value === void 0 || value.length === 0) {
|
|
337
|
-
throw invalid(`Attribute '${name}' is missing or empty`);
|
|
338
|
-
}
|
|
339
|
-
return value;
|
|
340
|
-
}
|
|
341
|
-
function assertValidType(type, label) {
|
|
342
|
-
try {
|
|
343
|
-
assertEventType(type);
|
|
344
|
-
} catch (error) {
|
|
345
|
-
throw invalid(`${label} is not a valid event type: ${error.message}`);
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
function assertJsonDataContentType(dataContentType) {
|
|
349
|
-
const mediaType = mediaTypeOf(dataContentType);
|
|
350
|
-
if (mediaType !== void 0 && mediaType !== "application/json" && mediaType !== "text/json" && !mediaType.endsWith("+json")) {
|
|
351
|
-
throw invalid(`datacontenttype must be JSON, received '${dataContentType}'`);
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
362
|
function extensionAttributes(body) {
|
|
355
363
|
return Object.fromEntries(
|
|
356
364
|
Object.entries(body).filter(
|
|
@@ -396,7 +404,7 @@ function readKeyLeniently(key) {
|
|
|
396
404
|
return Buffer.from(key).toString("base64");
|
|
397
405
|
}
|
|
398
406
|
}
|
|
399
|
-
function resolveText(option, record, name) {
|
|
407
|
+
function resolveText(option, { record, name }) {
|
|
400
408
|
const value = typeof option === "function" ? option(record) : option;
|
|
401
409
|
if (value === void 0 || value === null) {
|
|
402
410
|
return void 0;
|
|
@@ -407,7 +415,7 @@ function resolveText(option, record, name) {
|
|
|
407
415
|
}
|
|
408
416
|
return value.length === 0 ? void 0 : value;
|
|
409
417
|
}
|
|
410
|
-
function resolveOccurredAt(option, plain, record) {
|
|
418
|
+
function resolveOccurredAt(option, { plain, record }) {
|
|
411
419
|
const value = option?.(plain);
|
|
412
420
|
if (value === void 0) {
|
|
413
421
|
return requireRecordTimestamp(record);
|
|
@@ -424,13 +432,70 @@ function observeRejection(value) {
|
|
|
424
432
|
}
|
|
425
433
|
}
|
|
426
434
|
|
|
427
|
-
// src/
|
|
428
|
-
var
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
435
|
+
// src/codecs.ts
|
|
436
|
+
var cloudEventsBinary = Object.freeze({
|
|
437
|
+
encode: encodeEnvelope,
|
|
438
|
+
decode: decodeEnvelope
|
|
439
|
+
});
|
|
440
|
+
var cloudEventsStructured = Object.freeze({
|
|
441
|
+
encode: encodeStructuredEnvelope,
|
|
442
|
+
decode: decodeCloudEvent
|
|
443
|
+
});
|
|
444
|
+
var RECORD_POSITION = "record-position";
|
|
445
|
+
var EVENT_ID_HEADER = "relaybus-event-id";
|
|
446
|
+
var DEFAULT_CONTENT_TYPE = "application/json";
|
|
447
|
+
function jsonEnvelope(mapping) {
|
|
448
|
+
const format = createJsonEnvelopeFormat(mapping, {
|
|
449
|
+
transportIds: [RECORD_POSITION],
|
|
450
|
+
failures: { invalid, unencodable: (reason) => new UnencodableEventError(reason) }
|
|
451
|
+
});
|
|
452
|
+
const contentType = mapping.contentType ?? DEFAULT_CONTENT_TYPE;
|
|
453
|
+
const stampsTime = mapping.occurredAt === void 0;
|
|
454
|
+
const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { [CONTENT_TYPE]: contentType, [EVENT_ID_HEADER]: envelope.id } : { [CONTENT_TYPE]: contentType };
|
|
455
|
+
return {
|
|
456
|
+
encode: (envelope) => {
|
|
457
|
+
assertRepresentable(envelope);
|
|
458
|
+
const { body, headers } = format.write(envelope);
|
|
459
|
+
const taken = [CONTENT_TYPE, EVENT_ID_HEADER].find((name) => Object.hasOwn(headers, name));
|
|
460
|
+
if (taken !== void 0) {
|
|
461
|
+
throw new UnencodableEventError(
|
|
462
|
+
`Event ${envelope.id} has a header '${taken}', which the codec writes itself`
|
|
463
|
+
);
|
|
464
|
+
}
|
|
465
|
+
return {
|
|
466
|
+
key: envelope.key,
|
|
467
|
+
value: serializeJson(envelope, body),
|
|
468
|
+
headers: { ...headers, ...ownHeaders(envelope) },
|
|
469
|
+
...stampsTime ? { timestamp: recordTimestamp(envelope) } : {}
|
|
470
|
+
};
|
|
471
|
+
},
|
|
472
|
+
decode: (record) => {
|
|
473
|
+
const { [EVENT_ID_HEADER]: eventId, ...headers } = withoutContentType(
|
|
474
|
+
readHeadersLeniently(record.headers)
|
|
475
|
+
);
|
|
476
|
+
const fields = format.read({ body: parsePayload(record.value), headers });
|
|
477
|
+
return {
|
|
478
|
+
...fields,
|
|
479
|
+
id: fields.id ?? (eventId === void 0 || eventId === "" ? `kafka:${record.topic}:${record.partition}:${record.offset}` : eventId)
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
function withoutContentType(headers) {
|
|
485
|
+
return Object.fromEntries(Object.entries(headers).filter(([name]) => name !== CONTENT_TYPE));
|
|
486
|
+
}
|
|
487
|
+
function assertRepresentable(envelope) {
|
|
488
|
+
const kind = typeof envelope.payload;
|
|
489
|
+
if (kind === "undefined" || kind === "function" || kind === "symbol") {
|
|
490
|
+
throw new UnencodableEventError(
|
|
491
|
+
`Event ${envelope.id} has a payload that JSON cannot represent (undefined, a function or a symbol)`
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
function recordTimestamp(envelope) {
|
|
496
|
+
formatTime(envelope);
|
|
497
|
+
return String(envelope.occurredAt.getTime());
|
|
498
|
+
}
|
|
434
499
|
var DEFAULTS = {
|
|
435
500
|
name: "kafka",
|
|
436
501
|
maxInFlightRequests: 1,
|
|
@@ -445,6 +510,7 @@ var DEFAULTS = {
|
|
|
445
510
|
};
|
|
446
511
|
var KafkaTransport = class {
|
|
447
512
|
name;
|
|
513
|
+
system = "kafka";
|
|
448
514
|
#kafka;
|
|
449
515
|
#groupId;
|
|
450
516
|
#fromBeginning;
|
|
@@ -452,7 +518,7 @@ var KafkaTransport = class {
|
|
|
452
518
|
#producerConfig;
|
|
453
519
|
#consumerConfig;
|
|
454
520
|
#invalidPolicy;
|
|
455
|
-
#
|
|
521
|
+
#codec;
|
|
456
522
|
#logger;
|
|
457
523
|
#settings;
|
|
458
524
|
#sleepers = /* @__PURE__ */ new Set();
|
|
@@ -471,16 +537,13 @@ var KafkaTransport = class {
|
|
|
471
537
|
this.#kafka = options.kafka;
|
|
472
538
|
this.#groupId = options.groupId;
|
|
473
539
|
this.#fromBeginning = options.fromBeginning ?? true;
|
|
474
|
-
this.#maxInFlightRequests =
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
`maxInFlightRequests must be an integer from 1 to 5, received ${this.#maxInFlightRequests}`
|
|
478
|
-
);
|
|
479
|
-
}
|
|
540
|
+
this.#maxInFlightRequests = assertMaxInFlightRequests(
|
|
541
|
+
options.maxInFlightRequests ?? DEFAULTS.maxInFlightRequests
|
|
542
|
+
);
|
|
480
543
|
this.#producerConfig = options.producerConfig ?? {};
|
|
481
544
|
this.#consumerConfig = options.consumerConfig ?? {};
|
|
482
545
|
this.#invalidPolicy = options.onInvalidMessage ?? "dead-letter";
|
|
483
|
-
this.#
|
|
546
|
+
this.#codec = resolveCodec(options);
|
|
484
547
|
this.#logger = options.logger ?? console;
|
|
485
548
|
this.#settings = resolveSettings(options);
|
|
486
549
|
}
|
|
@@ -510,11 +573,10 @@ var KafkaTransport = class {
|
|
|
510
573
|
}
|
|
511
574
|
async publish(envelope, destination) {
|
|
512
575
|
const producer = await this.#connectedProducer();
|
|
513
|
-
const { key, value, headers } = encodeEnvelope(envelope);
|
|
514
576
|
await this.#send(producer, {
|
|
515
577
|
topic: destination,
|
|
516
578
|
acks: -1,
|
|
517
|
-
messages: [
|
|
579
|
+
messages: [this.#codec.encode(envelope)]
|
|
518
580
|
});
|
|
519
581
|
}
|
|
520
582
|
/**
|
|
@@ -524,13 +586,14 @@ var KafkaTransport = class {
|
|
|
524
586
|
* @throws ConfigurationError when `groupId` is missing, `destinations` is empty
|
|
525
587
|
* or the transport already subscribed (create one transport per subscription).
|
|
526
588
|
*/
|
|
527
|
-
|
|
589
|
+
// eslint-disable-next-line relaybus/max-two-params -- the Transport.subscribe signature
|
|
590
|
+
async subscribe(destinations, onMessage, hooks) {
|
|
528
591
|
this.#assertCanSubscribe(destinations);
|
|
529
592
|
this.#state = "subscribed";
|
|
530
593
|
const consumer = this.#kafka.consumer({ ...this.#consumerConfig, groupId: this.#groupId });
|
|
531
594
|
this.#consumer = consumer;
|
|
532
595
|
this.#observeConsumer(consumer);
|
|
533
|
-
const start = () => this.#startConsumer(consumer, [...destinations], onMessage);
|
|
596
|
+
const start = () => this.#startConsumer(consumer, { destinations: [...destinations], onMessage, hooks });
|
|
534
597
|
this.#startConsuming = start;
|
|
535
598
|
try {
|
|
536
599
|
await start();
|
|
@@ -546,12 +609,18 @@ var KafkaTransport = class {
|
|
|
546
609
|
`[KafkaTransport](subscribe) Consuming | transport=${this.name} group=${this.#groupId} topics=${destinations.join(",")}`
|
|
547
610
|
);
|
|
548
611
|
}
|
|
549
|
-
async #startConsumer(consumer,
|
|
612
|
+
async #startConsumer(consumer, {
|
|
613
|
+
destinations,
|
|
614
|
+
onMessage,
|
|
615
|
+
hooks
|
|
616
|
+
}) {
|
|
550
617
|
await consumer.connect();
|
|
551
618
|
await consumer.subscribe({ topics: destinations, fromBeginning: this.#fromBeginning });
|
|
552
619
|
const joined = this.#waitForGroupJoin(consumer);
|
|
553
620
|
try {
|
|
554
|
-
await consumer.run({
|
|
621
|
+
await consumer.run({
|
|
622
|
+
eachMessage: (payload) => this.#handleMessage(payload, { onMessage, hooks })
|
|
623
|
+
});
|
|
555
624
|
await joined.promise;
|
|
556
625
|
} finally {
|
|
557
626
|
joined.dispose();
|
|
@@ -569,7 +638,7 @@ var KafkaTransport = class {
|
|
|
569
638
|
for (const wake of this.#sleepers) {
|
|
570
639
|
wake();
|
|
571
640
|
}
|
|
572
|
-
this.#joinWaiter?.reject(new
|
|
641
|
+
this.#joinWaiter?.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
|
|
573
642
|
await this.#restartLoop;
|
|
574
643
|
const producer = await pendingProducer?.catch(() => void 0);
|
|
575
644
|
const results = await Promise.allSettled([
|
|
@@ -583,7 +652,7 @@ var KafkaTransport = class {
|
|
|
583
652
|
}
|
|
584
653
|
#assertCanSubscribe(destinations) {
|
|
585
654
|
if (this.#state === "closed") {
|
|
586
|
-
throw new
|
|
655
|
+
throw new TransportClosedError(`Transport '${this.name}' is closed`);
|
|
587
656
|
}
|
|
588
657
|
if (this.#state === "subscribed") {
|
|
589
658
|
throw new ConfigurationError(
|
|
@@ -619,27 +688,30 @@ var KafkaTransport = class {
|
|
|
619
688
|
}
|
|
620
689
|
/** Sends with the publish timeout; the late outcome of a timed-out send is ignored. */
|
|
621
690
|
async #send(producer, record) {
|
|
622
|
-
let
|
|
691
|
+
let cancel;
|
|
623
692
|
const timeout = new Promise((_resolve, reject) => {
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
693
|
+
cancel = schedule(systemClock, {
|
|
694
|
+
run: () => {
|
|
695
|
+
reject(
|
|
696
|
+
new PublishTimeoutError(
|
|
697
|
+
`Transport '${this.name}' got no broker ack for topic '${record.topic}' within ${this.#settings.publishTimeoutMs}ms`
|
|
698
|
+
)
|
|
699
|
+
);
|
|
700
|
+
},
|
|
701
|
+
delayMs: this.#settings.publishTimeoutMs
|
|
702
|
+
});
|
|
631
703
|
});
|
|
632
704
|
const sending = producer.send(record);
|
|
633
705
|
sending.catch(() => void 0);
|
|
634
706
|
try {
|
|
635
707
|
await Promise.race([sending, timeout]);
|
|
636
708
|
} finally {
|
|
637
|
-
|
|
709
|
+
cancel?.();
|
|
638
710
|
}
|
|
639
711
|
}
|
|
640
712
|
#connectedProducer() {
|
|
641
713
|
if (this.#state === "closed") {
|
|
642
|
-
return Promise.reject(new
|
|
714
|
+
return Promise.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
|
|
643
715
|
}
|
|
644
716
|
this.#producer ??= this.#openProducer();
|
|
645
717
|
return this.#producer;
|
|
@@ -659,31 +731,40 @@ var KafkaTransport = class {
|
|
|
659
731
|
this.#producerConnected = this.#state !== "closed";
|
|
660
732
|
return producer;
|
|
661
733
|
}
|
|
662
|
-
async #handleMessage(payload,
|
|
734
|
+
async #handleMessage(payload, {
|
|
735
|
+
onMessage,
|
|
736
|
+
hooks
|
|
737
|
+
}) {
|
|
663
738
|
let envelope;
|
|
664
739
|
try {
|
|
665
740
|
envelope = this.#decodeRecord(payload);
|
|
666
741
|
} catch (error) {
|
|
667
|
-
if (this.#
|
|
742
|
+
if (this.#codec === cloudEventsBinary && !(error instanceof InvalidKafkaMessageError)) {
|
|
668
743
|
throw toHandlerFailure(error);
|
|
669
744
|
}
|
|
670
|
-
|
|
671
|
-
await this.#withRetries(payload,
|
|
745
|
+
hooks?.onInvalidMessage({ destination: payload.topic, error });
|
|
746
|
+
await this.#withRetries(payload, {
|
|
747
|
+
method: "handleInvalid",
|
|
748
|
+
operation: () => this.#handleInvalid(payload, error)
|
|
749
|
+
});
|
|
672
750
|
return;
|
|
673
751
|
}
|
|
674
|
-
await this.#withRetries(payload,
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
752
|
+
await this.#withRetries(payload, {
|
|
753
|
+
method: "deliver",
|
|
754
|
+
operation: async () => {
|
|
755
|
+
try {
|
|
756
|
+
await onMessage(envelope, { destination: payload.topic });
|
|
757
|
+
} catch (error) {
|
|
758
|
+
if (!(error instanceof NonRetryableError)) {
|
|
759
|
+
throw error;
|
|
760
|
+
}
|
|
761
|
+
await this.#handleInvalid(payload, error);
|
|
680
762
|
}
|
|
681
|
-
await this.#handleInvalid(payload, error.message);
|
|
682
763
|
}
|
|
683
764
|
});
|
|
684
765
|
}
|
|
685
766
|
#decodeRecord({ topic, partition, message }) {
|
|
686
|
-
if (this.#
|
|
767
|
+
if (this.#codec === cloudEventsBinary) {
|
|
687
768
|
return decodeEnvelope(message);
|
|
688
769
|
}
|
|
689
770
|
const record = {
|
|
@@ -695,16 +776,18 @@ var KafkaTransport = class {
|
|
|
695
776
|
headers: message.headers,
|
|
696
777
|
timestamp: message.timestamp
|
|
697
778
|
};
|
|
698
|
-
return completeDecodedEvent(this.#decode(record), record);
|
|
779
|
+
return completeDecodedEvent(this.#codec.decode(record), record);
|
|
699
780
|
}
|
|
700
|
-
async #handleInvalid(payload,
|
|
781
|
+
async #handleInvalid(payload, error) {
|
|
701
782
|
const { topic, partition, message } = payload;
|
|
783
|
+
const { message: reason, name, code } = toErrorInfo(error);
|
|
702
784
|
const location = `topic=${topic} partition=${partition} offset=${message.offset}`;
|
|
785
|
+
const errorFields = `errorName=${name}${code === void 0 ? "" : ` code=${code}`}`;
|
|
703
786
|
const policy = this.#invalidPolicy;
|
|
704
787
|
const deadLetterTopic = this.#deadLetterTopicFor(topic);
|
|
705
788
|
if (policy === "skip") {
|
|
706
789
|
this.#logger.error(
|
|
707
|
-
`[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason}`
|
|
790
|
+
`[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason} ${errorFields}`
|
|
708
791
|
);
|
|
709
792
|
} else if (typeof policy === "function") {
|
|
710
793
|
await policy({
|
|
@@ -716,6 +799,19 @@ var KafkaTransport = class {
|
|
|
716
799
|
value: message.value
|
|
717
800
|
});
|
|
718
801
|
} else {
|
|
802
|
+
const headers = {
|
|
803
|
+
...toOutgoingHeaders(message.headers),
|
|
804
|
+
"relaybus-invalid-reason": reason,
|
|
805
|
+
"relaybus-source-topic": topic,
|
|
806
|
+
"relaybus-source-partition": String(partition),
|
|
807
|
+
"relaybus-source-offset": message.offset,
|
|
808
|
+
"relaybus-error-name": name
|
|
809
|
+
};
|
|
810
|
+
if (code === void 0) {
|
|
811
|
+
delete headers["relaybus-error-code"];
|
|
812
|
+
} else {
|
|
813
|
+
headers["relaybus-error-code"] = code;
|
|
814
|
+
}
|
|
719
815
|
const producer = await this.#connectedProducer();
|
|
720
816
|
await this.#send(producer, {
|
|
721
817
|
topic: deadLetterTopic,
|
|
@@ -724,18 +820,12 @@ var KafkaTransport = class {
|
|
|
724
820
|
{
|
|
725
821
|
key: message.key === null ? null : Buffer.from(message.key),
|
|
726
822
|
value: message.value === null ? null : Buffer.from(message.value),
|
|
727
|
-
headers
|
|
728
|
-
...toOutgoingHeaders(message.headers),
|
|
729
|
-
"relaybus-invalid-reason": reason,
|
|
730
|
-
"relaybus-source-topic": topic,
|
|
731
|
-
"relaybus-source-partition": String(partition),
|
|
732
|
-
"relaybus-source-offset": message.offset
|
|
733
|
-
}
|
|
823
|
+
headers
|
|
734
824
|
}
|
|
735
825
|
]
|
|
736
826
|
});
|
|
737
827
|
this.#logger.error(
|
|
738
|
-
`[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason}`
|
|
828
|
+
`[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason} ${errorFields}`
|
|
739
829
|
);
|
|
740
830
|
}
|
|
741
831
|
}
|
|
@@ -744,7 +834,7 @@ var KafkaTransport = class {
|
|
|
744
834
|
* last error is rethrown so the client leaves the offset uncommitted and
|
|
745
835
|
* redelivers the record; heartbeats between attempts keep the session alive.
|
|
746
836
|
*/
|
|
747
|
-
async #withRetries(payload, method, operation) {
|
|
837
|
+
async #withRetries(payload, { method, operation }) {
|
|
748
838
|
const { maxAttempts, initialBackoffMs, maxBackoffMs } = this.#settings;
|
|
749
839
|
const location = `topic=${payload.topic} partition=${payload.partition} offset=${payload.message.offset}`;
|
|
750
840
|
const partitionKey = `${payload.topic}:${payload.partition}`;
|
|
@@ -758,12 +848,12 @@ var KafkaTransport = class {
|
|
|
758
848
|
if (attempt >= maxAttempts || this.#state === "closed") {
|
|
759
849
|
this.#stalledPartitions.add(partitionKey);
|
|
760
850
|
this.#logger.error(
|
|
761
|
-
`[KafkaTransport](
|
|
851
|
+
`[KafkaTransport](withRetries) Giving up for now, the record stays uncommitted and will be redelivered | operation=${method} ${location} attempts=${attempt} error=${reason}`
|
|
762
852
|
);
|
|
763
853
|
throw toHandlerFailure(error);
|
|
764
854
|
}
|
|
765
855
|
this.#logger.warn(
|
|
766
|
-
`[KafkaTransport](
|
|
856
|
+
`[KafkaTransport](withRetries) Attempt failed, retrying | operation=${method} ${location} attempt=${attempt} error=${reason}`
|
|
767
857
|
);
|
|
768
858
|
await this.#sleepWithHeartbeats(
|
|
769
859
|
Math.min(initialBackoffMs * 2 ** (attempt - 1), maxBackoffMs),
|
|
@@ -785,11 +875,11 @@ var KafkaTransport = class {
|
|
|
785
875
|
#sleep(milliseconds) {
|
|
786
876
|
return new Promise((resolve) => {
|
|
787
877
|
const wake = () => {
|
|
788
|
-
|
|
878
|
+
cancel();
|
|
789
879
|
this.#sleepers.delete(wake);
|
|
790
880
|
resolve();
|
|
791
881
|
};
|
|
792
|
-
const
|
|
882
|
+
const cancel = schedule(systemClock, { run: wake, delayMs: milliseconds });
|
|
793
883
|
this.#sleepers.add(wake);
|
|
794
884
|
});
|
|
795
885
|
}
|
|
@@ -871,42 +961,53 @@ var KafkaTransport = class {
|
|
|
871
961
|
if (!this.#tracksMembership(consumer)) {
|
|
872
962
|
return { promise: Promise.resolve(), dispose: () => void 0 };
|
|
873
963
|
}
|
|
874
|
-
let
|
|
964
|
+
let cancel;
|
|
875
965
|
const promise = new Promise((resolve, reject) => {
|
|
876
966
|
this.#joinWaiter = { resolve, reject };
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
967
|
+
cancel = schedule(systemClock, {
|
|
968
|
+
run: () => {
|
|
969
|
+
reject(
|
|
970
|
+
new GroupJoinTimeoutError(
|
|
971
|
+
`Consumer of transport '${this.name}' did not join group '${this.#groupId}' within ${this.#settings.startupTimeoutMs}ms`
|
|
972
|
+
)
|
|
973
|
+
);
|
|
974
|
+
},
|
|
975
|
+
delayMs: this.#settings.startupTimeoutMs
|
|
976
|
+
});
|
|
884
977
|
});
|
|
885
978
|
promise.catch(() => void 0);
|
|
886
979
|
return {
|
|
887
980
|
promise,
|
|
888
981
|
dispose: () => {
|
|
889
|
-
|
|
982
|
+
cancel?.();
|
|
890
983
|
this.#joinWaiter = void 0;
|
|
891
984
|
}
|
|
892
985
|
};
|
|
893
986
|
}
|
|
894
987
|
};
|
|
895
|
-
function
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
);
|
|
988
|
+
function resolveCodec({ codec, decode }) {
|
|
989
|
+
if (decode === void 0) {
|
|
990
|
+
return codec ?? cloudEventsBinary;
|
|
991
|
+
}
|
|
992
|
+
if (codec !== void 0) {
|
|
993
|
+
throw new ConfigurationError("Give 'codec' or the deprecated 'decode', not both");
|
|
901
994
|
}
|
|
995
|
+
return { encode: encodeEnvelope, decode };
|
|
996
|
+
}
|
|
997
|
+
function resolveSettings(options) {
|
|
998
|
+
const processing = options.processing ?? {};
|
|
999
|
+
const maxAttempts = assertPositiveInteger(
|
|
1000
|
+
processing.maxAttempts ?? DEFAULTS.maxAttempts,
|
|
1001
|
+
"processing.maxAttempts"
|
|
1002
|
+
);
|
|
902
1003
|
return {
|
|
903
1004
|
maxAttempts,
|
|
904
1005
|
initialBackoffMs: toMilliseconds(
|
|
905
|
-
|
|
1006
|
+
processing.initialBackoff ?? DEFAULTS.initialBackoff,
|
|
906
1007
|
"processing.initialBackoff"
|
|
907
1008
|
),
|
|
908
1009
|
maxBackoffMs: toMilliseconds(
|
|
909
|
-
|
|
1010
|
+
processing.maxBackoff ?? DEFAULTS.maxBackoff,
|
|
910
1011
|
"processing.maxBackoff"
|
|
911
1012
|
),
|
|
912
1013
|
startupTimeoutMs: toMilliseconds(
|
|
@@ -920,52 +1021,39 @@ function resolveSettings(options) {
|
|
|
920
1021
|
heartbeatIntervalMs: Math.max(
|
|
921
1022
|
1,
|
|
922
1023
|
toMilliseconds(
|
|
923
|
-
|
|
1024
|
+
processing.heartbeatInterval ?? DEFAULTS.heartbeatInterval,
|
|
924
1025
|
"processing.heartbeatInterval"
|
|
925
1026
|
)
|
|
926
1027
|
),
|
|
1028
|
+
...resolveRestartSettings(options.restart ?? {})
|
|
1029
|
+
};
|
|
1030
|
+
}
|
|
1031
|
+
function resolveRestartSettings(restart) {
|
|
1032
|
+
return {
|
|
927
1033
|
restartInitialBackoffMs: toMilliseconds(
|
|
928
|
-
|
|
1034
|
+
restart.initialBackoff ?? DEFAULTS.restartInitialBackoff,
|
|
929
1035
|
"restart.initialBackoff"
|
|
930
1036
|
),
|
|
931
1037
|
restartMaxBackoffMs: toMilliseconds(
|
|
932
|
-
|
|
1038
|
+
restart.maxBackoff ?? DEFAULTS.restartMaxBackoff,
|
|
933
1039
|
"restart.maxBackoff"
|
|
934
1040
|
)
|
|
935
1041
|
};
|
|
936
1042
|
}
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
}
|
|
941
|
-
function fromHeader(name) {
|
|
942
|
-
if (name.length === 0) {
|
|
943
|
-
throw new ConfigurationError("fromHeader needs a header name");
|
|
944
|
-
}
|
|
945
|
-
return (message) => Object.hasOwn(message.headers, name) ? message.headers[name] : void 0;
|
|
946
|
-
}
|
|
947
|
-
function fromPayloadField(path) {
|
|
948
|
-
const segments = path.split(".");
|
|
949
|
-
if (segments.some((segment) => segment.length === 0)) {
|
|
1043
|
+
var MAXIMUM_IN_FLIGHT_REQUESTS = 5;
|
|
1044
|
+
function assertMaxInFlightRequests(value) {
|
|
1045
|
+
if (!Number.isInteger(value) || value < 1 || value > MAXIMUM_IN_FLIGHT_REQUESTS) {
|
|
950
1046
|
throw new ConfigurationError(
|
|
951
|
-
`
|
|
1047
|
+
`maxInFlightRequests must be an integer from 1 to 5, received ${value}`
|
|
952
1048
|
);
|
|
953
1049
|
}
|
|
954
|
-
return
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
}
|
|
960
|
-
value = value[segment];
|
|
961
|
-
}
|
|
962
|
-
if (typeof value === "string") {
|
|
963
|
-
return value;
|
|
964
|
-
}
|
|
965
|
-
return Number.isSafeInteger(value) ? String(value) : void 0;
|
|
966
|
-
};
|
|
1050
|
+
return value;
|
|
1051
|
+
}
|
|
1052
|
+
function toHandlerFailure(error) {
|
|
1053
|
+
const { name, message } = toErrorInfo(error);
|
|
1054
|
+
return new HandlerFailedError(`${name}: ${message}`);
|
|
967
1055
|
}
|
|
968
1056
|
|
|
969
|
-
export { HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, UnencodableEventError, decodeCloudEvent, decodeEnvelope, encodeEnvelope,
|
|
1057
|
+
export { GroupJoinTimeoutError, HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, TransportClosedError, UnencodableEventError, cloudEventsBinary, cloudEventsStructured, decodeCloudEvent, decodeEnvelope, encodeEnvelope, jsonEnvelope, plainJsonDecoder };
|
|
970
1058
|
//# sourceMappingURL=index.js.map
|
|
971
1059
|
//# sourceMappingURL=index.js.map
|