@relaybus/kafka 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -48
- package/dist/index.cjs +267 -169
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +123 -62
- package/dist/index.d.ts +123 -62
- package/dist/index.js +256 -168
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -1,16 +1,37 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var core = require('@relaybus/core');
|
|
4
|
+
var internal = require('@relaybus/core/internal');
|
|
4
5
|
|
|
5
|
-
// src/
|
|
6
|
+
// src/errors.ts
|
|
6
7
|
var InvalidKafkaMessageError = class extends core.RelayBusError {
|
|
8
|
+
name = "InvalidKafkaMessageError";
|
|
9
|
+
code = "RELAYBUS_KAFKA_INVALID_MESSAGE";
|
|
7
10
|
};
|
|
8
11
|
var UnencodableEventError = class extends core.NonRetryableError {
|
|
9
12
|
name = "UnencodableEventError";
|
|
13
|
+
code = "RELAYBUS_KAFKA_UNENCODABLE_EVENT";
|
|
14
|
+
};
|
|
15
|
+
var PublishTimeoutError = class extends core.RelayBusError {
|
|
16
|
+
name = "PublishTimeoutError";
|
|
17
|
+
code = "RELAYBUS_KAFKA_PUBLISH_TIMEOUT";
|
|
18
|
+
};
|
|
19
|
+
var TransportClosedError = class extends core.RelayBusError {
|
|
20
|
+
name = "TransportClosedError";
|
|
21
|
+
code = "RELAYBUS_KAFKA_TRANSPORT_CLOSED";
|
|
22
|
+
};
|
|
23
|
+
var GroupJoinTimeoutError = class extends core.RelayBusError {
|
|
24
|
+
name = "GroupJoinTimeoutError";
|
|
25
|
+
code = "RELAYBUS_KAFKA_GROUP_JOIN_TIMEOUT";
|
|
26
|
+
};
|
|
27
|
+
var HandlerFailedError = class extends core.RelayBusError {
|
|
28
|
+
name = "HandlerFailedError";
|
|
29
|
+
code = "RELAYBUS_KAFKA_HANDLER_FAILED";
|
|
10
30
|
};
|
|
11
31
|
var SPEC_VERSION = "1.0";
|
|
12
32
|
var CONTENT_TYPE = "content-type";
|
|
13
33
|
var JSON_MEDIA_TYPE = "application/json";
|
|
34
|
+
var STRUCTURED_MEDIA_TYPE = "application/cloudevents+json";
|
|
14
35
|
var CLOUDEVENTS_PREFIX = "ce_";
|
|
15
36
|
var ESCAPE_PREFIX = "x-relaybus-h-";
|
|
16
37
|
var RESERVED_PREFIXES = [CLOUDEVENTS_PREFIX, "x-relaybus-"];
|
|
@@ -25,6 +46,7 @@ var HEADER = {
|
|
|
25
46
|
causationId: "ce_causationid"
|
|
26
47
|
};
|
|
27
48
|
var PROTOCOL_HEADERS = /* @__PURE__ */ new Set([...Object.values(HEADER), CONTENT_TYPE]);
|
|
49
|
+
var MAXIMUM_RFC3339_YEAR = 9999;
|
|
28
50
|
var RFC3339_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/i;
|
|
29
51
|
var strictUtf8 = new TextDecoder("utf-8", { fatal: true });
|
|
30
52
|
var lenientUtf8 = new TextDecoder("utf-8");
|
|
@@ -52,25 +74,39 @@ function encodeEnvelope(envelope) {
|
|
|
52
74
|
}
|
|
53
75
|
return { key: envelope.key, value, headers: Object.fromEntries(entries) };
|
|
54
76
|
}
|
|
77
|
+
function encodeStructuredEnvelope(envelope) {
|
|
78
|
+
const value = internal.writeStructuredCloudEvent(envelope, {
|
|
79
|
+
time: formatTime(envelope),
|
|
80
|
+
data: serializePayload(envelope)
|
|
81
|
+
});
|
|
82
|
+
const entries = Object.entries(envelope.headers).map(
|
|
83
|
+
([name, headerValue]) => [escapeUserHeaderName(name), headerValue]
|
|
84
|
+
);
|
|
85
|
+
entries.push([CONTENT_TYPE, STRUCTURED_MEDIA_TYPE]);
|
|
86
|
+
return { key: envelope.key, value, headers: Object.fromEntries(entries) };
|
|
87
|
+
}
|
|
55
88
|
function serializePayload(envelope) {
|
|
56
|
-
|
|
89
|
+
return serializeJson(envelope, envelope.payload);
|
|
90
|
+
}
|
|
91
|
+
function serializeJson(envelope, value) {
|
|
92
|
+
let text;
|
|
57
93
|
try {
|
|
58
|
-
|
|
94
|
+
text = JSON.stringify(value);
|
|
59
95
|
} catch (error) {
|
|
60
96
|
throw new UnencodableEventError(
|
|
61
97
|
`Event ${envelope.id} has a payload that JSON cannot represent: ${error.message}`
|
|
62
98
|
);
|
|
63
99
|
}
|
|
64
|
-
if (
|
|
100
|
+
if (text === void 0) {
|
|
65
101
|
throw new UnencodableEventError(
|
|
66
102
|
`Event ${envelope.id} has a payload that JSON cannot represent (undefined, a function or a symbol)`
|
|
67
103
|
);
|
|
68
104
|
}
|
|
69
|
-
return
|
|
105
|
+
return text;
|
|
70
106
|
}
|
|
71
107
|
function formatTime(envelope) {
|
|
72
108
|
const year = envelope.occurredAt.getUTCFullYear();
|
|
73
|
-
if (Number.isNaN(year) || year < 0 || year >
|
|
109
|
+
if (Number.isNaN(year) || year < 0 || year > MAXIMUM_RFC3339_YEAR) {
|
|
74
110
|
throw new UnencodableEventError(
|
|
75
111
|
`Event ${envelope.id} has an occurredAt that is not an RFC 3339 instant (invalid date or year outside 0..9999)`
|
|
76
112
|
);
|
|
@@ -118,11 +154,14 @@ function readHeaders(incoming) {
|
|
|
118
154
|
if (typeof raw !== "string" && !(raw instanceof Uint8Array)) {
|
|
119
155
|
throw invalid(`Header '${name}' is repeated`);
|
|
120
156
|
}
|
|
121
|
-
headers.set(
|
|
157
|
+
headers.set(
|
|
158
|
+
name,
|
|
159
|
+
decodeText(raw, { strict: PROTOCOL_HEADERS.has(name), label: `Header '${name}'` })
|
|
160
|
+
);
|
|
122
161
|
}
|
|
123
162
|
return headers;
|
|
124
163
|
}
|
|
125
|
-
function decodeText(raw, strict, label) {
|
|
164
|
+
function decodeText(raw, { strict, label }) {
|
|
126
165
|
if (typeof raw === "string") {
|
|
127
166
|
return raw;
|
|
128
167
|
}
|
|
@@ -145,19 +184,19 @@ function assertJsonContentType(contentType) {
|
|
|
145
184
|
throw invalid(`${CONTENT_TYPE} must be '${JSON_MEDIA_TYPE}', received '${contentType ?? ""}'`);
|
|
146
185
|
}
|
|
147
186
|
}
|
|
148
|
-
function resolveTime(text,
|
|
187
|
+
function resolveTime(text, recordTimestamp2) {
|
|
149
188
|
if (text !== void 0 && text.length > 0) {
|
|
150
189
|
return parseTime(text);
|
|
151
190
|
}
|
|
152
|
-
const time = readRecordTimestamp(
|
|
191
|
+
const time = readRecordTimestamp(recordTimestamp2);
|
|
153
192
|
if (time === void 0) {
|
|
154
193
|
throw invalid(`${HEADER.time} is missing and the record has no usable timestamp`);
|
|
155
194
|
}
|
|
156
195
|
return time;
|
|
157
196
|
}
|
|
158
|
-
function readRecordTimestamp(
|
|
159
|
-
const time = new Date(Number(
|
|
160
|
-
return
|
|
197
|
+
function readRecordTimestamp(recordTimestamp2) {
|
|
198
|
+
const time = new Date(Number(recordTimestamp2));
|
|
199
|
+
return recordTimestamp2 === void 0 || recordTimestamp2 === "" || Number.isNaN(time.getTime()) ? void 0 : time;
|
|
161
200
|
}
|
|
162
201
|
function parseTime(text, label = HEADER.time) {
|
|
163
202
|
const time = new Date(text);
|
|
@@ -170,7 +209,7 @@ function parsePayload(value) {
|
|
|
170
209
|
if (value === null) {
|
|
171
210
|
throw invalid("Message has no value");
|
|
172
211
|
}
|
|
173
|
-
const text = decodeText(value, true, "Message value");
|
|
212
|
+
const text = decodeText(value, { strict: true, label: "Message value" });
|
|
174
213
|
try {
|
|
175
214
|
return JSON.parse(text);
|
|
176
215
|
} catch {
|
|
@@ -185,7 +224,7 @@ function resolveKey(subject, recordKey) {
|
|
|
185
224
|
return subject ?? keyText;
|
|
186
225
|
}
|
|
187
226
|
function readRecordKey(recordKey) {
|
|
188
|
-
return recordKey === null ? null : decodeText(recordKey, true, "Message key");
|
|
227
|
+
return recordKey === null ? null : decodeText(recordKey, { strict: true, label: "Message key" });
|
|
189
228
|
}
|
|
190
229
|
function collectUserHeaders(headers) {
|
|
191
230
|
const entries = [];
|
|
@@ -214,7 +253,8 @@ function toOutgoingHeaders(incoming) {
|
|
|
214
253
|
function toBufferOrString(value) {
|
|
215
254
|
return typeof value === "string" ? value : Buffer.from(value);
|
|
216
255
|
}
|
|
217
|
-
var
|
|
256
|
+
var { optionalAttribute, requiredAttribute, assertValidType, assertJsonDataContentType } = internal.createCloudEventReaders(invalid);
|
|
257
|
+
var STRUCTURED_MEDIA_TYPE2 = "application/cloudevents+json";
|
|
218
258
|
var exactUtf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
219
259
|
var SPEC_VERSION2 = "1.0";
|
|
220
260
|
var SPEC_VERSION_HEADER = "ce_specversion";
|
|
@@ -234,11 +274,11 @@ var STRUCTURED_ATTRIBUTES = /* @__PURE__ */ new Set([
|
|
|
234
274
|
]);
|
|
235
275
|
function decodeCloudEvent(record) {
|
|
236
276
|
const headers = readHeaders(record.headers);
|
|
237
|
-
if (mediaTypeOf(headers.get(CONTENT_TYPE)) !==
|
|
277
|
+
if (internal.mediaTypeOf(headers.get(CONTENT_TYPE)) !== STRUCTURED_MEDIA_TYPE2) {
|
|
238
278
|
return decodeEnvelope(record);
|
|
239
279
|
}
|
|
240
280
|
const body = parsePayload(record.value);
|
|
241
|
-
if (!isJsonObject(body)) {
|
|
281
|
+
if (!internal.isJsonObject(body)) {
|
|
242
282
|
throw invalid("A structured CloudEvent must be a JSON object");
|
|
243
283
|
}
|
|
244
284
|
if (body["specversion"] !== SPEC_VERSION2) {
|
|
@@ -269,7 +309,7 @@ function plainJsonDecoder(options) {
|
|
|
269
309
|
}
|
|
270
310
|
return (record) => {
|
|
271
311
|
const headers = readHeadersLeniently(record.headers);
|
|
272
|
-
if (record.headers?.[SPEC_VERSION_HEADER] !== void 0 || mediaTypeOf(headers[CONTENT_TYPE]) ===
|
|
312
|
+
if (record.headers?.[SPEC_VERSION_HEADER] !== void 0 || internal.mediaTypeOf(headers[CONTENT_TYPE]) === STRUCTURED_MEDIA_TYPE2) {
|
|
273
313
|
return decodeCloudEvent(record);
|
|
274
314
|
}
|
|
275
315
|
const plain = {
|
|
@@ -281,21 +321,21 @@ function plainJsonDecoder(options) {
|
|
|
281
321
|
payload: parsePayload(record.value),
|
|
282
322
|
timestamp: record.timestamp
|
|
283
323
|
};
|
|
284
|
-
const type = resolveText(options.type, plain, "type");
|
|
324
|
+
const type = resolveText(options.type, { record: plain, name: "type" });
|
|
285
325
|
if (type === void 0) {
|
|
286
326
|
throw invalid("Record has no event type: the type option found none");
|
|
287
327
|
}
|
|
288
328
|
assertValidType(type, "The event type");
|
|
289
329
|
return {
|
|
290
|
-
id: resolveText(options.id, plain, "id") ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
|
|
330
|
+
id: resolveText(options.id, { record: plain, name: "id" }) ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
|
|
291
331
|
type,
|
|
292
|
-
source: resolveText(options.source, plain, "source") ?? record.topic,
|
|
293
|
-
key: options.key === void 0 ? plain.key : resolveText(options.key, plain, "key") ?? null,
|
|
332
|
+
source: resolveText(options.source, { record: plain, name: "source" }) ?? record.topic,
|
|
333
|
+
key: options.key === void 0 ? plain.key : resolveText(options.key, { record: plain, name: "key" }) ?? null,
|
|
294
334
|
payload: plain.payload,
|
|
295
335
|
headers,
|
|
296
336
|
correlationId: null,
|
|
297
337
|
causationId: null,
|
|
298
|
-
occurredAt: resolveOccurredAt(options.occurredAt, plain, record)
|
|
338
|
+
occurredAt: resolveOccurredAt(options.occurredAt, { plain, record })
|
|
299
339
|
};
|
|
300
340
|
};
|
|
301
341
|
}
|
|
@@ -320,39 +360,6 @@ function completeDecodedEvent(decoded, record) {
|
|
|
320
360
|
occurredAt: event.occurredAt ?? requireRecordTimestamp(record)
|
|
321
361
|
};
|
|
322
362
|
}
|
|
323
|
-
function mediaTypeOf(contentType) {
|
|
324
|
-
return contentType?.split(";")[0]?.trim().toLowerCase();
|
|
325
|
-
}
|
|
326
|
-
function isJsonObject(value) {
|
|
327
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
328
|
-
}
|
|
329
|
-
function optionalAttribute(body, name) {
|
|
330
|
-
const value = body[name];
|
|
331
|
-
if (value !== void 0 && typeof value !== "string") {
|
|
332
|
-
throw invalid(`Attribute '${name}' must be a string`);
|
|
333
|
-
}
|
|
334
|
-
return value;
|
|
335
|
-
}
|
|
336
|
-
function requiredAttribute(body, name) {
|
|
337
|
-
const value = optionalAttribute(body, name);
|
|
338
|
-
if (value === void 0 || value.length === 0) {
|
|
339
|
-
throw invalid(`Attribute '${name}' is missing or empty`);
|
|
340
|
-
}
|
|
341
|
-
return value;
|
|
342
|
-
}
|
|
343
|
-
function assertValidType(type, label) {
|
|
344
|
-
try {
|
|
345
|
-
core.assertEventType(type);
|
|
346
|
-
} catch (error) {
|
|
347
|
-
throw invalid(`${label} is not a valid event type: ${error.message}`);
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
function assertJsonDataContentType(dataContentType) {
|
|
351
|
-
const mediaType = mediaTypeOf(dataContentType);
|
|
352
|
-
if (mediaType !== void 0 && mediaType !== "application/json" && mediaType !== "text/json" && !mediaType.endsWith("+json")) {
|
|
353
|
-
throw invalid(`datacontenttype must be JSON, received '${dataContentType}'`);
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
363
|
function extensionAttributes(body) {
|
|
357
364
|
return Object.fromEntries(
|
|
358
365
|
Object.entries(body).filter(
|
|
@@ -398,7 +405,7 @@ function readKeyLeniently(key) {
|
|
|
398
405
|
return Buffer.from(key).toString("base64");
|
|
399
406
|
}
|
|
400
407
|
}
|
|
401
|
-
function resolveText(option, record, name) {
|
|
408
|
+
function resolveText(option, { record, name }) {
|
|
402
409
|
const value = typeof option === "function" ? option(record) : option;
|
|
403
410
|
if (value === void 0 || value === null) {
|
|
404
411
|
return void 0;
|
|
@@ -409,7 +416,7 @@ function resolveText(option, record, name) {
|
|
|
409
416
|
}
|
|
410
417
|
return value.length === 0 ? void 0 : value;
|
|
411
418
|
}
|
|
412
|
-
function resolveOccurredAt(option, plain, record) {
|
|
419
|
+
function resolveOccurredAt(option, { plain, record }) {
|
|
413
420
|
const value = option?.(plain);
|
|
414
421
|
if (value === void 0) {
|
|
415
422
|
return requireRecordTimestamp(record);
|
|
@@ -426,13 +433,70 @@ function observeRejection(value) {
|
|
|
426
433
|
}
|
|
427
434
|
}
|
|
428
435
|
|
|
429
|
-
// src/
|
|
430
|
-
var
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
+
// src/codecs.ts
|
|
437
|
+
var cloudEventsBinary = Object.freeze({
|
|
438
|
+
encode: encodeEnvelope,
|
|
439
|
+
decode: decodeEnvelope
|
|
440
|
+
});
|
|
441
|
+
var cloudEventsStructured = Object.freeze({
|
|
442
|
+
encode: encodeStructuredEnvelope,
|
|
443
|
+
decode: decodeCloudEvent
|
|
444
|
+
});
|
|
445
|
+
var RECORD_POSITION = "record-position";
|
|
446
|
+
var EVENT_ID_HEADER = "relaybus-event-id";
|
|
447
|
+
var DEFAULT_CONTENT_TYPE = "application/json";
|
|
448
|
+
function jsonEnvelope(mapping) {
|
|
449
|
+
const format = internal.createJsonEnvelopeFormat(mapping, {
|
|
450
|
+
transportIds: [RECORD_POSITION],
|
|
451
|
+
failures: { invalid, unencodable: (reason) => new UnencodableEventError(reason) }
|
|
452
|
+
});
|
|
453
|
+
const contentType = mapping.contentType ?? DEFAULT_CONTENT_TYPE;
|
|
454
|
+
const stampsTime = mapping.occurredAt === void 0;
|
|
455
|
+
const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { [CONTENT_TYPE]: contentType, [EVENT_ID_HEADER]: envelope.id } : { [CONTENT_TYPE]: contentType };
|
|
456
|
+
return {
|
|
457
|
+
encode: (envelope) => {
|
|
458
|
+
assertRepresentable(envelope);
|
|
459
|
+
const { body, headers } = format.write(envelope);
|
|
460
|
+
const taken = [CONTENT_TYPE, EVENT_ID_HEADER].find((name) => Object.hasOwn(headers, name));
|
|
461
|
+
if (taken !== void 0) {
|
|
462
|
+
throw new UnencodableEventError(
|
|
463
|
+
`Event ${envelope.id} has a header '${taken}', which the codec writes itself`
|
|
464
|
+
);
|
|
465
|
+
}
|
|
466
|
+
return {
|
|
467
|
+
key: envelope.key,
|
|
468
|
+
value: serializeJson(envelope, body),
|
|
469
|
+
headers: { ...headers, ...ownHeaders(envelope) },
|
|
470
|
+
...stampsTime ? { timestamp: recordTimestamp(envelope) } : {}
|
|
471
|
+
};
|
|
472
|
+
},
|
|
473
|
+
decode: (record) => {
|
|
474
|
+
const { [EVENT_ID_HEADER]: eventId, ...headers } = withoutContentType(
|
|
475
|
+
readHeadersLeniently(record.headers)
|
|
476
|
+
);
|
|
477
|
+
const fields = format.read({ body: parsePayload(record.value), headers });
|
|
478
|
+
return {
|
|
479
|
+
...fields,
|
|
480
|
+
id: fields.id ?? (eventId === void 0 || eventId === "" ? `kafka:${record.topic}:${record.partition}:${record.offset}` : eventId)
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
function withoutContentType(headers) {
|
|
486
|
+
return Object.fromEntries(Object.entries(headers).filter(([name]) => name !== CONTENT_TYPE));
|
|
487
|
+
}
|
|
488
|
+
function assertRepresentable(envelope) {
|
|
489
|
+
const kind = typeof envelope.payload;
|
|
490
|
+
if (kind === "undefined" || kind === "function" || kind === "symbol") {
|
|
491
|
+
throw new UnencodableEventError(
|
|
492
|
+
`Event ${envelope.id} has a payload that JSON cannot represent (undefined, a function or a symbol)`
|
|
493
|
+
);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
function recordTimestamp(envelope) {
|
|
497
|
+
formatTime(envelope);
|
|
498
|
+
return String(envelope.occurredAt.getTime());
|
|
499
|
+
}
|
|
436
500
|
var DEFAULTS = {
|
|
437
501
|
name: "kafka",
|
|
438
502
|
maxInFlightRequests: 1,
|
|
@@ -447,6 +511,7 @@ var DEFAULTS = {
|
|
|
447
511
|
};
|
|
448
512
|
var KafkaTransport = class {
|
|
449
513
|
name;
|
|
514
|
+
system = "kafka";
|
|
450
515
|
#kafka;
|
|
451
516
|
#groupId;
|
|
452
517
|
#fromBeginning;
|
|
@@ -454,7 +519,7 @@ var KafkaTransport = class {
|
|
|
454
519
|
#producerConfig;
|
|
455
520
|
#consumerConfig;
|
|
456
521
|
#invalidPolicy;
|
|
457
|
-
#
|
|
522
|
+
#codec;
|
|
458
523
|
#logger;
|
|
459
524
|
#settings;
|
|
460
525
|
#sleepers = /* @__PURE__ */ new Set();
|
|
@@ -473,16 +538,13 @@ var KafkaTransport = class {
|
|
|
473
538
|
this.#kafka = options.kafka;
|
|
474
539
|
this.#groupId = options.groupId;
|
|
475
540
|
this.#fromBeginning = options.fromBeginning ?? true;
|
|
476
|
-
this.#maxInFlightRequests =
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
`maxInFlightRequests must be an integer from 1 to 5, received ${this.#maxInFlightRequests}`
|
|
480
|
-
);
|
|
481
|
-
}
|
|
541
|
+
this.#maxInFlightRequests = assertMaxInFlightRequests(
|
|
542
|
+
options.maxInFlightRequests ?? DEFAULTS.maxInFlightRequests
|
|
543
|
+
);
|
|
482
544
|
this.#producerConfig = options.producerConfig ?? {};
|
|
483
545
|
this.#consumerConfig = options.consumerConfig ?? {};
|
|
484
546
|
this.#invalidPolicy = options.onInvalidMessage ?? "dead-letter";
|
|
485
|
-
this.#
|
|
547
|
+
this.#codec = resolveCodec(options);
|
|
486
548
|
this.#logger = options.logger ?? console;
|
|
487
549
|
this.#settings = resolveSettings(options);
|
|
488
550
|
}
|
|
@@ -512,11 +574,10 @@ var KafkaTransport = class {
|
|
|
512
574
|
}
|
|
513
575
|
async publish(envelope, destination) {
|
|
514
576
|
const producer = await this.#connectedProducer();
|
|
515
|
-
const { key, value, headers } = encodeEnvelope(envelope);
|
|
516
577
|
await this.#send(producer, {
|
|
517
578
|
topic: destination,
|
|
518
579
|
acks: -1,
|
|
519
|
-
messages: [
|
|
580
|
+
messages: [this.#codec.encode(envelope)]
|
|
520
581
|
});
|
|
521
582
|
}
|
|
522
583
|
/**
|
|
@@ -526,13 +587,14 @@ var KafkaTransport = class {
|
|
|
526
587
|
* @throws ConfigurationError when `groupId` is missing, `destinations` is empty
|
|
527
588
|
* or the transport already subscribed (create one transport per subscription).
|
|
528
589
|
*/
|
|
529
|
-
|
|
590
|
+
// eslint-disable-next-line relaybus/max-two-params -- the Transport.subscribe signature
|
|
591
|
+
async subscribe(destinations, onMessage, hooks) {
|
|
530
592
|
this.#assertCanSubscribe(destinations);
|
|
531
593
|
this.#state = "subscribed";
|
|
532
594
|
const consumer = this.#kafka.consumer({ ...this.#consumerConfig, groupId: this.#groupId });
|
|
533
595
|
this.#consumer = consumer;
|
|
534
596
|
this.#observeConsumer(consumer);
|
|
535
|
-
const start = () => this.#startConsumer(consumer, [...destinations], onMessage);
|
|
597
|
+
const start = () => this.#startConsumer(consumer, { destinations: [...destinations], onMessage, hooks });
|
|
536
598
|
this.#startConsuming = start;
|
|
537
599
|
try {
|
|
538
600
|
await start();
|
|
@@ -548,12 +610,18 @@ var KafkaTransport = class {
|
|
|
548
610
|
`[KafkaTransport](subscribe) Consuming | transport=${this.name} group=${this.#groupId} topics=${destinations.join(",")}`
|
|
549
611
|
);
|
|
550
612
|
}
|
|
551
|
-
async #startConsumer(consumer,
|
|
613
|
+
async #startConsumer(consumer, {
|
|
614
|
+
destinations,
|
|
615
|
+
onMessage,
|
|
616
|
+
hooks
|
|
617
|
+
}) {
|
|
552
618
|
await consumer.connect();
|
|
553
619
|
await consumer.subscribe({ topics: destinations, fromBeginning: this.#fromBeginning });
|
|
554
620
|
const joined = this.#waitForGroupJoin(consumer);
|
|
555
621
|
try {
|
|
556
|
-
await consumer.run({
|
|
622
|
+
await consumer.run({
|
|
623
|
+
eachMessage: (payload) => this.#handleMessage(payload, { onMessage, hooks })
|
|
624
|
+
});
|
|
557
625
|
await joined.promise;
|
|
558
626
|
} finally {
|
|
559
627
|
joined.dispose();
|
|
@@ -571,7 +639,7 @@ var KafkaTransport = class {
|
|
|
571
639
|
for (const wake of this.#sleepers) {
|
|
572
640
|
wake();
|
|
573
641
|
}
|
|
574
|
-
this.#joinWaiter?.reject(new
|
|
642
|
+
this.#joinWaiter?.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
|
|
575
643
|
await this.#restartLoop;
|
|
576
644
|
const producer = await pendingProducer?.catch(() => void 0);
|
|
577
645
|
const results = await Promise.allSettled([
|
|
@@ -585,7 +653,7 @@ var KafkaTransport = class {
|
|
|
585
653
|
}
|
|
586
654
|
#assertCanSubscribe(destinations) {
|
|
587
655
|
if (this.#state === "closed") {
|
|
588
|
-
throw new
|
|
656
|
+
throw new TransportClosedError(`Transport '${this.name}' is closed`);
|
|
589
657
|
}
|
|
590
658
|
if (this.#state === "subscribed") {
|
|
591
659
|
throw new core.ConfigurationError(
|
|
@@ -621,27 +689,30 @@ var KafkaTransport = class {
|
|
|
621
689
|
}
|
|
622
690
|
/** Sends with the publish timeout; the late outcome of a timed-out send is ignored. */
|
|
623
691
|
async #send(producer, record) {
|
|
624
|
-
let
|
|
692
|
+
let cancel;
|
|
625
693
|
const timeout = new Promise((_resolve, reject) => {
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
694
|
+
cancel = internal.schedule(core.systemClock, {
|
|
695
|
+
run: () => {
|
|
696
|
+
reject(
|
|
697
|
+
new PublishTimeoutError(
|
|
698
|
+
`Transport '${this.name}' got no broker ack for topic '${record.topic}' within ${this.#settings.publishTimeoutMs}ms`
|
|
699
|
+
)
|
|
700
|
+
);
|
|
701
|
+
},
|
|
702
|
+
delayMs: this.#settings.publishTimeoutMs
|
|
703
|
+
});
|
|
633
704
|
});
|
|
634
705
|
const sending = producer.send(record);
|
|
635
706
|
sending.catch(() => void 0);
|
|
636
707
|
try {
|
|
637
708
|
await Promise.race([sending, timeout]);
|
|
638
709
|
} finally {
|
|
639
|
-
|
|
710
|
+
cancel?.();
|
|
640
711
|
}
|
|
641
712
|
}
|
|
642
713
|
#connectedProducer() {
|
|
643
714
|
if (this.#state === "closed") {
|
|
644
|
-
return Promise.reject(new
|
|
715
|
+
return Promise.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
|
|
645
716
|
}
|
|
646
717
|
this.#producer ??= this.#openProducer();
|
|
647
718
|
return this.#producer;
|
|
@@ -661,31 +732,40 @@ var KafkaTransport = class {
|
|
|
661
732
|
this.#producerConnected = this.#state !== "closed";
|
|
662
733
|
return producer;
|
|
663
734
|
}
|
|
664
|
-
async #handleMessage(payload,
|
|
735
|
+
async #handleMessage(payload, {
|
|
736
|
+
onMessage,
|
|
737
|
+
hooks
|
|
738
|
+
}) {
|
|
665
739
|
let envelope;
|
|
666
740
|
try {
|
|
667
741
|
envelope = this.#decodeRecord(payload);
|
|
668
742
|
} catch (error) {
|
|
669
|
-
if (this.#
|
|
743
|
+
if (this.#codec === cloudEventsBinary && !(error instanceof InvalidKafkaMessageError)) {
|
|
670
744
|
throw toHandlerFailure(error);
|
|
671
745
|
}
|
|
672
|
-
|
|
673
|
-
await this.#withRetries(payload,
|
|
746
|
+
hooks?.onInvalidMessage({ destination: payload.topic, error });
|
|
747
|
+
await this.#withRetries(payload, {
|
|
748
|
+
method: "handleInvalid",
|
|
749
|
+
operation: () => this.#handleInvalid(payload, error)
|
|
750
|
+
});
|
|
674
751
|
return;
|
|
675
752
|
}
|
|
676
|
-
await this.#withRetries(payload,
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
753
|
+
await this.#withRetries(payload, {
|
|
754
|
+
method: "deliver",
|
|
755
|
+
operation: async () => {
|
|
756
|
+
try {
|
|
757
|
+
await onMessage(envelope, { destination: payload.topic });
|
|
758
|
+
} catch (error) {
|
|
759
|
+
if (!(error instanceof core.NonRetryableError)) {
|
|
760
|
+
throw error;
|
|
761
|
+
}
|
|
762
|
+
await this.#handleInvalid(payload, error);
|
|
682
763
|
}
|
|
683
|
-
await this.#handleInvalid(payload, error.message);
|
|
684
764
|
}
|
|
685
765
|
});
|
|
686
766
|
}
|
|
687
767
|
#decodeRecord({ topic, partition, message }) {
|
|
688
|
-
if (this.#
|
|
768
|
+
if (this.#codec === cloudEventsBinary) {
|
|
689
769
|
return decodeEnvelope(message);
|
|
690
770
|
}
|
|
691
771
|
const record = {
|
|
@@ -697,16 +777,18 @@ var KafkaTransport = class {
|
|
|
697
777
|
headers: message.headers,
|
|
698
778
|
timestamp: message.timestamp
|
|
699
779
|
};
|
|
700
|
-
return completeDecodedEvent(this.#decode(record), record);
|
|
780
|
+
return completeDecodedEvent(this.#codec.decode(record), record);
|
|
701
781
|
}
|
|
702
|
-
async #handleInvalid(payload,
|
|
782
|
+
async #handleInvalid(payload, error) {
|
|
703
783
|
const { topic, partition, message } = payload;
|
|
784
|
+
const { message: reason, name, code } = core.toErrorInfo(error);
|
|
704
785
|
const location = `topic=${topic} partition=${partition} offset=${message.offset}`;
|
|
786
|
+
const errorFields = `errorName=${name}${code === void 0 ? "" : ` code=${code}`}`;
|
|
705
787
|
const policy = this.#invalidPolicy;
|
|
706
788
|
const deadLetterTopic = this.#deadLetterTopicFor(topic);
|
|
707
789
|
if (policy === "skip") {
|
|
708
790
|
this.#logger.error(
|
|
709
|
-
`[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason}`
|
|
791
|
+
`[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason} ${errorFields}`
|
|
710
792
|
);
|
|
711
793
|
} else if (typeof policy === "function") {
|
|
712
794
|
await policy({
|
|
@@ -718,6 +800,19 @@ var KafkaTransport = class {
|
|
|
718
800
|
value: message.value
|
|
719
801
|
});
|
|
720
802
|
} else {
|
|
803
|
+
const headers = {
|
|
804
|
+
...toOutgoingHeaders(message.headers),
|
|
805
|
+
"relaybus-invalid-reason": reason,
|
|
806
|
+
"relaybus-source-topic": topic,
|
|
807
|
+
"relaybus-source-partition": String(partition),
|
|
808
|
+
"relaybus-source-offset": message.offset,
|
|
809
|
+
"relaybus-error-name": name
|
|
810
|
+
};
|
|
811
|
+
if (code === void 0) {
|
|
812
|
+
delete headers["relaybus-error-code"];
|
|
813
|
+
} else {
|
|
814
|
+
headers["relaybus-error-code"] = code;
|
|
815
|
+
}
|
|
721
816
|
const producer = await this.#connectedProducer();
|
|
722
817
|
await this.#send(producer, {
|
|
723
818
|
topic: deadLetterTopic,
|
|
@@ -726,18 +821,12 @@ var KafkaTransport = class {
|
|
|
726
821
|
{
|
|
727
822
|
key: message.key === null ? null : Buffer.from(message.key),
|
|
728
823
|
value: message.value === null ? null : Buffer.from(message.value),
|
|
729
|
-
headers
|
|
730
|
-
...toOutgoingHeaders(message.headers),
|
|
731
|
-
"relaybus-invalid-reason": reason,
|
|
732
|
-
"relaybus-source-topic": topic,
|
|
733
|
-
"relaybus-source-partition": String(partition),
|
|
734
|
-
"relaybus-source-offset": message.offset
|
|
735
|
-
}
|
|
824
|
+
headers
|
|
736
825
|
}
|
|
737
826
|
]
|
|
738
827
|
});
|
|
739
828
|
this.#logger.error(
|
|
740
|
-
`[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason}`
|
|
829
|
+
`[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason} ${errorFields}`
|
|
741
830
|
);
|
|
742
831
|
}
|
|
743
832
|
}
|
|
@@ -746,7 +835,7 @@ var KafkaTransport = class {
|
|
|
746
835
|
* last error is rethrown so the client leaves the offset uncommitted and
|
|
747
836
|
* redelivers the record; heartbeats between attempts keep the session alive.
|
|
748
837
|
*/
|
|
749
|
-
async #withRetries(payload, method, operation) {
|
|
838
|
+
async #withRetries(payload, { method, operation }) {
|
|
750
839
|
const { maxAttempts, initialBackoffMs, maxBackoffMs } = this.#settings;
|
|
751
840
|
const location = `topic=${payload.topic} partition=${payload.partition} offset=${payload.message.offset}`;
|
|
752
841
|
const partitionKey = `${payload.topic}:${payload.partition}`;
|
|
@@ -760,12 +849,12 @@ var KafkaTransport = class {
|
|
|
760
849
|
if (attempt >= maxAttempts || this.#state === "closed") {
|
|
761
850
|
this.#stalledPartitions.add(partitionKey);
|
|
762
851
|
this.#logger.error(
|
|
763
|
-
`[KafkaTransport](
|
|
852
|
+
`[KafkaTransport](withRetries) Giving up for now, the record stays uncommitted and will be redelivered | operation=${method} ${location} attempts=${attempt} error=${reason}`
|
|
764
853
|
);
|
|
765
854
|
throw toHandlerFailure(error);
|
|
766
855
|
}
|
|
767
856
|
this.#logger.warn(
|
|
768
|
-
`[KafkaTransport](
|
|
857
|
+
`[KafkaTransport](withRetries) Attempt failed, retrying | operation=${method} ${location} attempt=${attempt} error=${reason}`
|
|
769
858
|
);
|
|
770
859
|
await this.#sleepWithHeartbeats(
|
|
771
860
|
Math.min(initialBackoffMs * 2 ** (attempt - 1), maxBackoffMs),
|
|
@@ -787,11 +876,11 @@ var KafkaTransport = class {
|
|
|
787
876
|
#sleep(milliseconds) {
|
|
788
877
|
return new Promise((resolve) => {
|
|
789
878
|
const wake = () => {
|
|
790
|
-
|
|
879
|
+
cancel();
|
|
791
880
|
this.#sleepers.delete(wake);
|
|
792
881
|
resolve();
|
|
793
882
|
};
|
|
794
|
-
const
|
|
883
|
+
const cancel = internal.schedule(core.systemClock, { run: wake, delayMs: milliseconds });
|
|
795
884
|
this.#sleepers.add(wake);
|
|
796
885
|
});
|
|
797
886
|
}
|
|
@@ -873,42 +962,53 @@ var KafkaTransport = class {
|
|
|
873
962
|
if (!this.#tracksMembership(consumer)) {
|
|
874
963
|
return { promise: Promise.resolve(), dispose: () => void 0 };
|
|
875
964
|
}
|
|
876
|
-
let
|
|
965
|
+
let cancel;
|
|
877
966
|
const promise = new Promise((resolve, reject) => {
|
|
878
967
|
this.#joinWaiter = { resolve, reject };
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
968
|
+
cancel = internal.schedule(core.systemClock, {
|
|
969
|
+
run: () => {
|
|
970
|
+
reject(
|
|
971
|
+
new GroupJoinTimeoutError(
|
|
972
|
+
`Consumer of transport '${this.name}' did not join group '${this.#groupId}' within ${this.#settings.startupTimeoutMs}ms`
|
|
973
|
+
)
|
|
974
|
+
);
|
|
975
|
+
},
|
|
976
|
+
delayMs: this.#settings.startupTimeoutMs
|
|
977
|
+
});
|
|
886
978
|
});
|
|
887
979
|
promise.catch(() => void 0);
|
|
888
980
|
return {
|
|
889
981
|
promise,
|
|
890
982
|
dispose: () => {
|
|
891
|
-
|
|
983
|
+
cancel?.();
|
|
892
984
|
this.#joinWaiter = void 0;
|
|
893
985
|
}
|
|
894
986
|
};
|
|
895
987
|
}
|
|
896
988
|
};
|
|
897
|
-
function
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
);
|
|
989
|
+
function resolveCodec({ codec, decode }) {
|
|
990
|
+
if (decode === void 0) {
|
|
991
|
+
return codec ?? cloudEventsBinary;
|
|
992
|
+
}
|
|
993
|
+
if (codec !== void 0) {
|
|
994
|
+
throw new core.ConfigurationError("Give 'codec' or the deprecated 'decode', not both");
|
|
903
995
|
}
|
|
996
|
+
return { encode: encodeEnvelope, decode };
|
|
997
|
+
}
|
|
998
|
+
function resolveSettings(options) {
|
|
999
|
+
const processing = options.processing ?? {};
|
|
1000
|
+
const maxAttempts = internal.assertPositiveInteger(
|
|
1001
|
+
processing.maxAttempts ?? DEFAULTS.maxAttempts,
|
|
1002
|
+
"processing.maxAttempts"
|
|
1003
|
+
);
|
|
904
1004
|
return {
|
|
905
1005
|
maxAttempts,
|
|
906
1006
|
initialBackoffMs: core.toMilliseconds(
|
|
907
|
-
|
|
1007
|
+
processing.initialBackoff ?? DEFAULTS.initialBackoff,
|
|
908
1008
|
"processing.initialBackoff"
|
|
909
1009
|
),
|
|
910
1010
|
maxBackoffMs: core.toMilliseconds(
|
|
911
|
-
|
|
1011
|
+
processing.maxBackoff ?? DEFAULTS.maxBackoff,
|
|
912
1012
|
"processing.maxBackoff"
|
|
913
1013
|
),
|
|
914
1014
|
startupTimeoutMs: core.toMilliseconds(
|
|
@@ -922,62 +1022,60 @@ function resolveSettings(options) {
|
|
|
922
1022
|
heartbeatIntervalMs: Math.max(
|
|
923
1023
|
1,
|
|
924
1024
|
core.toMilliseconds(
|
|
925
|
-
|
|
1025
|
+
processing.heartbeatInterval ?? DEFAULTS.heartbeatInterval,
|
|
926
1026
|
"processing.heartbeatInterval"
|
|
927
1027
|
)
|
|
928
1028
|
),
|
|
1029
|
+
...resolveRestartSettings(options.restart ?? {})
|
|
1030
|
+
};
|
|
1031
|
+
}
|
|
1032
|
+
function resolveRestartSettings(restart) {
|
|
1033
|
+
return {
|
|
929
1034
|
restartInitialBackoffMs: core.toMilliseconds(
|
|
930
|
-
|
|
1035
|
+
restart.initialBackoff ?? DEFAULTS.restartInitialBackoff,
|
|
931
1036
|
"restart.initialBackoff"
|
|
932
1037
|
),
|
|
933
1038
|
restartMaxBackoffMs: core.toMilliseconds(
|
|
934
|
-
|
|
1039
|
+
restart.maxBackoff ?? DEFAULTS.restartMaxBackoff,
|
|
935
1040
|
"restart.maxBackoff"
|
|
936
1041
|
)
|
|
937
1042
|
};
|
|
938
1043
|
}
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
}
|
|
943
|
-
function fromHeader(name) {
|
|
944
|
-
if (name.length === 0) {
|
|
945
|
-
throw new core.ConfigurationError("fromHeader needs a header name");
|
|
946
|
-
}
|
|
947
|
-
return (message) => Object.hasOwn(message.headers, name) ? message.headers[name] : void 0;
|
|
948
|
-
}
|
|
949
|
-
function fromPayloadField(path) {
|
|
950
|
-
const segments = path.split(".");
|
|
951
|
-
if (segments.some((segment) => segment.length === 0)) {
|
|
1044
|
+
var MAXIMUM_IN_FLIGHT_REQUESTS = 5;
|
|
1045
|
+
function assertMaxInFlightRequests(value) {
|
|
1046
|
+
if (!Number.isInteger(value) || value < 1 || value > MAXIMUM_IN_FLIGHT_REQUESTS) {
|
|
952
1047
|
throw new core.ConfigurationError(
|
|
953
|
-
`
|
|
1048
|
+
`maxInFlightRequests must be an integer from 1 to 5, received ${value}`
|
|
954
1049
|
);
|
|
955
1050
|
}
|
|
956
|
-
return
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
}
|
|
962
|
-
value = value[segment];
|
|
963
|
-
}
|
|
964
|
-
if (typeof value === "string") {
|
|
965
|
-
return value;
|
|
966
|
-
}
|
|
967
|
-
return Number.isSafeInteger(value) ? String(value) : void 0;
|
|
968
|
-
};
|
|
1051
|
+
return value;
|
|
1052
|
+
}
|
|
1053
|
+
function toHandlerFailure(error) {
|
|
1054
|
+
const { name, message } = core.toErrorInfo(error);
|
|
1055
|
+
return new HandlerFailedError(`${name}: ${message}`);
|
|
969
1056
|
}
|
|
970
1057
|
|
|
1058
|
+
Object.defineProperty(exports, "fromHeader", {
|
|
1059
|
+
enumerable: true,
|
|
1060
|
+
get: function () { return internal.fromHeader; }
|
|
1061
|
+
});
|
|
1062
|
+
Object.defineProperty(exports, "fromPayloadField", {
|
|
1063
|
+
enumerable: true,
|
|
1064
|
+
get: function () { return internal.fromPayloadField; }
|
|
1065
|
+
});
|
|
1066
|
+
exports.GroupJoinTimeoutError = GroupJoinTimeoutError;
|
|
971
1067
|
exports.HandlerFailedError = HandlerFailedError;
|
|
972
1068
|
exports.InvalidKafkaMessageError = InvalidKafkaMessageError;
|
|
973
1069
|
exports.KafkaTransport = KafkaTransport;
|
|
974
1070
|
exports.PublishTimeoutError = PublishTimeoutError;
|
|
1071
|
+
exports.TransportClosedError = TransportClosedError;
|
|
975
1072
|
exports.UnencodableEventError = UnencodableEventError;
|
|
1073
|
+
exports.cloudEventsBinary = cloudEventsBinary;
|
|
1074
|
+
exports.cloudEventsStructured = cloudEventsStructured;
|
|
976
1075
|
exports.decodeCloudEvent = decodeCloudEvent;
|
|
977
1076
|
exports.decodeEnvelope = decodeEnvelope;
|
|
978
1077
|
exports.encodeEnvelope = encodeEnvelope;
|
|
979
|
-
exports.
|
|
980
|
-
exports.fromPayloadField = fromPayloadField;
|
|
1078
|
+
exports.jsonEnvelope = jsonEnvelope;
|
|
981
1079
|
exports.plainJsonDecoder = plainJsonDecoder;
|
|
982
1080
|
//# sourceMappingURL=index.cjs.map
|
|
983
1081
|
//# sourceMappingURL=index.cjs.map
|