@relaybus/kafka 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +166 -21
- package/dist/index.cjs +458 -96
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +245 -23
- package/dist/index.d.ts +245 -23
- package/dist/index.js +446 -98
- package/dist/index.js.map +1 -1
- package/package.json +22 -9
package/dist/index.js
CHANGED
|
@@ -1,14 +1,36 @@
|
|
|
1
|
-
import { RelayBusError, NonRetryableError, assertEventType, ConfigurationError, toErrorInfo, toMilliseconds } from '@relaybus/core';
|
|
1
|
+
import { RelayBusError, NonRetryableError, assertEventType, ConfigurationError, systemClock, toErrorInfo, toMilliseconds } from '@relaybus/core';
|
|
2
|
+
import { createCloudEventReaders, mediaTypeOf, isJsonObject, createJsonEnvelopeFormat, schedule, assertPositiveInteger, writeStructuredCloudEvent } from '@relaybus/core/internal';
|
|
3
|
+
export { fromHeader, fromPayloadField } from '@relaybus/core/internal';
|
|
2
4
|
|
|
3
|
-
// src/
|
|
5
|
+
// src/errors.ts
|
|
4
6
|
var InvalidKafkaMessageError = class extends RelayBusError {
|
|
7
|
+
name = "InvalidKafkaMessageError";
|
|
8
|
+
code = "RELAYBUS_KAFKA_INVALID_MESSAGE";
|
|
5
9
|
};
|
|
6
10
|
var UnencodableEventError = class extends NonRetryableError {
|
|
7
11
|
name = "UnencodableEventError";
|
|
12
|
+
code = "RELAYBUS_KAFKA_UNENCODABLE_EVENT";
|
|
13
|
+
};
|
|
14
|
+
var PublishTimeoutError = class extends RelayBusError {
|
|
15
|
+
name = "PublishTimeoutError";
|
|
16
|
+
code = "RELAYBUS_KAFKA_PUBLISH_TIMEOUT";
|
|
17
|
+
};
|
|
18
|
+
var TransportClosedError = class extends RelayBusError {
|
|
19
|
+
name = "TransportClosedError";
|
|
20
|
+
code = "RELAYBUS_KAFKA_TRANSPORT_CLOSED";
|
|
21
|
+
};
|
|
22
|
+
var GroupJoinTimeoutError = class extends RelayBusError {
|
|
23
|
+
name = "GroupJoinTimeoutError";
|
|
24
|
+
code = "RELAYBUS_KAFKA_GROUP_JOIN_TIMEOUT";
|
|
25
|
+
};
|
|
26
|
+
var HandlerFailedError = class extends RelayBusError {
|
|
27
|
+
name = "HandlerFailedError";
|
|
28
|
+
code = "RELAYBUS_KAFKA_HANDLER_FAILED";
|
|
8
29
|
};
|
|
9
30
|
var SPEC_VERSION = "1.0";
|
|
10
31
|
var CONTENT_TYPE = "content-type";
|
|
11
32
|
var JSON_MEDIA_TYPE = "application/json";
|
|
33
|
+
var STRUCTURED_MEDIA_TYPE = "application/cloudevents+json";
|
|
12
34
|
var CLOUDEVENTS_PREFIX = "ce_";
|
|
13
35
|
var ESCAPE_PREFIX = "x-relaybus-h-";
|
|
14
36
|
var RESERVED_PREFIXES = [CLOUDEVENTS_PREFIX, "x-relaybus-"];
|
|
@@ -23,6 +45,7 @@ var HEADER = {
|
|
|
23
45
|
causationId: "ce_causationid"
|
|
24
46
|
};
|
|
25
47
|
var PROTOCOL_HEADERS = /* @__PURE__ */ new Set([...Object.values(HEADER), CONTENT_TYPE]);
|
|
48
|
+
var MAXIMUM_RFC3339_YEAR = 9999;
|
|
26
49
|
var RFC3339_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/i;
|
|
27
50
|
var strictUtf8 = new TextDecoder("utf-8", { fatal: true });
|
|
28
51
|
var lenientUtf8 = new TextDecoder("utf-8");
|
|
@@ -50,25 +73,39 @@ function encodeEnvelope(envelope) {
|
|
|
50
73
|
}
|
|
51
74
|
return { key: envelope.key, value, headers: Object.fromEntries(entries) };
|
|
52
75
|
}
|
|
76
|
+
function encodeStructuredEnvelope(envelope) {
|
|
77
|
+
const value = writeStructuredCloudEvent(envelope, {
|
|
78
|
+
time: formatTime(envelope),
|
|
79
|
+
data: serializePayload(envelope)
|
|
80
|
+
});
|
|
81
|
+
const entries = Object.entries(envelope.headers).map(
|
|
82
|
+
([name, headerValue]) => [escapeUserHeaderName(name), headerValue]
|
|
83
|
+
);
|
|
84
|
+
entries.push([CONTENT_TYPE, STRUCTURED_MEDIA_TYPE]);
|
|
85
|
+
return { key: envelope.key, value, headers: Object.fromEntries(entries) };
|
|
86
|
+
}
|
|
53
87
|
function serializePayload(envelope) {
|
|
54
|
-
|
|
88
|
+
return serializeJson(envelope, envelope.payload);
|
|
89
|
+
}
|
|
90
|
+
function serializeJson(envelope, value) {
|
|
91
|
+
let text;
|
|
55
92
|
try {
|
|
56
|
-
|
|
93
|
+
text = JSON.stringify(value);
|
|
57
94
|
} catch (error) {
|
|
58
95
|
throw new UnencodableEventError(
|
|
59
96
|
`Event ${envelope.id} has a payload that JSON cannot represent: ${error.message}`
|
|
60
97
|
);
|
|
61
98
|
}
|
|
62
|
-
if (
|
|
99
|
+
if (text === void 0) {
|
|
63
100
|
throw new UnencodableEventError(
|
|
64
101
|
`Event ${envelope.id} has a payload that JSON cannot represent (undefined, a function or a symbol)`
|
|
65
102
|
);
|
|
66
103
|
}
|
|
67
|
-
return
|
|
104
|
+
return text;
|
|
68
105
|
}
|
|
69
106
|
function formatTime(envelope) {
|
|
70
107
|
const year = envelope.occurredAt.getUTCFullYear();
|
|
71
|
-
if (Number.isNaN(year) || year < 0 || year >
|
|
108
|
+
if (Number.isNaN(year) || year < 0 || year > MAXIMUM_RFC3339_YEAR) {
|
|
72
109
|
throw new UnencodableEventError(
|
|
73
110
|
`Event ${envelope.id} has an occurredAt that is not an RFC 3339 instant (invalid date or year outside 0..9999)`
|
|
74
111
|
);
|
|
@@ -116,11 +153,14 @@ function readHeaders(incoming) {
|
|
|
116
153
|
if (typeof raw !== "string" && !(raw instanceof Uint8Array)) {
|
|
117
154
|
throw invalid(`Header '${name}' is repeated`);
|
|
118
155
|
}
|
|
119
|
-
headers.set(
|
|
156
|
+
headers.set(
|
|
157
|
+
name,
|
|
158
|
+
decodeText(raw, { strict: PROTOCOL_HEADERS.has(name), label: `Header '${name}'` })
|
|
159
|
+
);
|
|
120
160
|
}
|
|
121
161
|
return headers;
|
|
122
162
|
}
|
|
123
|
-
function decodeText(raw, strict, label) {
|
|
163
|
+
function decodeText(raw, { strict, label }) {
|
|
124
164
|
if (typeof raw === "string") {
|
|
125
165
|
return raw;
|
|
126
166
|
}
|
|
@@ -143,21 +183,24 @@ function assertJsonContentType(contentType) {
|
|
|
143
183
|
throw invalid(`${CONTENT_TYPE} must be '${JSON_MEDIA_TYPE}', received '${contentType ?? ""}'`);
|
|
144
184
|
}
|
|
145
185
|
}
|
|
146
|
-
function resolveTime(text,
|
|
186
|
+
function resolveTime(text, recordTimestamp2) {
|
|
147
187
|
if (text !== void 0 && text.length > 0) {
|
|
148
188
|
return parseTime(text);
|
|
149
189
|
}
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
if (recordTimestamp === void 0 || recordTimestamp === "" || Number.isNaN(time.getTime())) {
|
|
190
|
+
const time = readRecordTimestamp(recordTimestamp2);
|
|
191
|
+
if (time === void 0) {
|
|
153
192
|
throw invalid(`${HEADER.time} is missing and the record has no usable timestamp`);
|
|
154
193
|
}
|
|
155
194
|
return time;
|
|
156
195
|
}
|
|
157
|
-
function
|
|
196
|
+
function readRecordTimestamp(recordTimestamp2) {
|
|
197
|
+
const time = new Date(Number(recordTimestamp2));
|
|
198
|
+
return recordTimestamp2 === void 0 || recordTimestamp2 === "" || Number.isNaN(time.getTime()) ? void 0 : time;
|
|
199
|
+
}
|
|
200
|
+
function parseTime(text, label = HEADER.time) {
|
|
158
201
|
const time = new Date(text);
|
|
159
202
|
if (!RFC3339_PATTERN.test(text) || Number.isNaN(time.getTime())) {
|
|
160
|
-
throw invalid(`${
|
|
203
|
+
throw invalid(`${label} is not an RFC 3339 timestamp: '${text}'`);
|
|
161
204
|
}
|
|
162
205
|
return time;
|
|
163
206
|
}
|
|
@@ -165,7 +208,7 @@ function parsePayload(value) {
|
|
|
165
208
|
if (value === null) {
|
|
166
209
|
throw invalid("Message has no value");
|
|
167
210
|
}
|
|
168
|
-
const text = decodeText(value, true, "Message value");
|
|
211
|
+
const text = decodeText(value, { strict: true, label: "Message value" });
|
|
169
212
|
try {
|
|
170
213
|
return JSON.parse(text);
|
|
171
214
|
} catch {
|
|
@@ -173,12 +216,15 @@ function parsePayload(value) {
|
|
|
173
216
|
}
|
|
174
217
|
}
|
|
175
218
|
function resolveKey(subject, recordKey) {
|
|
176
|
-
const keyText =
|
|
219
|
+
const keyText = readRecordKey(recordKey);
|
|
177
220
|
if (subject !== void 0 && keyText !== null && subject !== keyText) {
|
|
178
221
|
throw invalid(`${HEADER.subject} does not match the message key`);
|
|
179
222
|
}
|
|
180
223
|
return subject ?? keyText;
|
|
181
224
|
}
|
|
225
|
+
function readRecordKey(recordKey) {
|
|
226
|
+
return recordKey === null ? null : decodeText(recordKey, { strict: true, label: "Message key" });
|
|
227
|
+
}
|
|
182
228
|
function collectUserHeaders(headers) {
|
|
183
229
|
const entries = [];
|
|
184
230
|
for (const [name, value] of headers) {
|
|
@@ -206,12 +252,250 @@ function toOutgoingHeaders(incoming) {
|
|
|
206
252
|
function toBufferOrString(value) {
|
|
207
253
|
return typeof value === "string" ? value : Buffer.from(value);
|
|
208
254
|
}
|
|
209
|
-
var
|
|
210
|
-
|
|
211
|
-
};
|
|
212
|
-
var
|
|
213
|
-
|
|
214
|
-
|
|
255
|
+
var { optionalAttribute, requiredAttribute, assertValidType, assertJsonDataContentType } = createCloudEventReaders(invalid);
|
|
256
|
+
var STRUCTURED_MEDIA_TYPE2 = "application/cloudevents+json";
|
|
257
|
+
var exactUtf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
258
|
+
var SPEC_VERSION2 = "1.0";
|
|
259
|
+
var SPEC_VERSION_HEADER = "ce_specversion";
|
|
260
|
+
var STRUCTURED_ATTRIBUTES = /* @__PURE__ */ new Set([
|
|
261
|
+
"specversion",
|
|
262
|
+
"id",
|
|
263
|
+
"type",
|
|
264
|
+
"source",
|
|
265
|
+
"time",
|
|
266
|
+
"subject",
|
|
267
|
+
"datacontenttype",
|
|
268
|
+
"dataschema",
|
|
269
|
+
"data",
|
|
270
|
+
"data_base64",
|
|
271
|
+
"correlationid",
|
|
272
|
+
"causationid"
|
|
273
|
+
]);
|
|
274
|
+
function decodeCloudEvent(record) {
|
|
275
|
+
const headers = readHeaders(record.headers);
|
|
276
|
+
if (mediaTypeOf(headers.get(CONTENT_TYPE)) !== STRUCTURED_MEDIA_TYPE2) {
|
|
277
|
+
return decodeEnvelope(record);
|
|
278
|
+
}
|
|
279
|
+
const body = parsePayload(record.value);
|
|
280
|
+
if (!isJsonObject(body)) {
|
|
281
|
+
throw invalid("A structured CloudEvent must be a JSON object");
|
|
282
|
+
}
|
|
283
|
+
if (body["specversion"] !== SPEC_VERSION2) {
|
|
284
|
+
throw invalid(`specversion must be '${SPEC_VERSION2}'`);
|
|
285
|
+
}
|
|
286
|
+
const type = requiredAttribute(body, "type");
|
|
287
|
+
assertValidType(type, "type");
|
|
288
|
+
assertJsonDataContentType(optionalAttribute(body, "datacontenttype"));
|
|
289
|
+
if (Object.hasOwn(body, "data_base64")) {
|
|
290
|
+
throw invalid("data_base64 is not supported: relaybus payloads are JSON, send them as data");
|
|
291
|
+
}
|
|
292
|
+
const time = optionalAttribute(body, "time");
|
|
293
|
+
return {
|
|
294
|
+
id: requiredAttribute(body, "id"),
|
|
295
|
+
type,
|
|
296
|
+
source: requiredAttribute(body, "source"),
|
|
297
|
+
key: optionalAttribute(body, "subject") ?? readRecordKey(record.key),
|
|
298
|
+
payload: Object.hasOwn(body, "data") ? body["data"] : null,
|
|
299
|
+
headers: { ...collectUserHeaders(headers), ...extensionAttributes(body) },
|
|
300
|
+
correlationId: optionalAttribute(body, "correlationid") ?? null,
|
|
301
|
+
causationId: optionalAttribute(body, "causationid") ?? null,
|
|
302
|
+
occurredAt: time === void 0 ? requireRecordTimestamp(record) : parseTime(time, "time")
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
function plainJsonDecoder(options) {
|
|
306
|
+
if (typeof options.type === "string") {
|
|
307
|
+
assertEventType(options.type);
|
|
308
|
+
}
|
|
309
|
+
return (record) => {
|
|
310
|
+
const headers = readHeadersLeniently(record.headers);
|
|
311
|
+
if (record.headers?.[SPEC_VERSION_HEADER] !== void 0 || mediaTypeOf(headers[CONTENT_TYPE]) === STRUCTURED_MEDIA_TYPE2) {
|
|
312
|
+
return decodeCloudEvent(record);
|
|
313
|
+
}
|
|
314
|
+
const plain = {
|
|
315
|
+
topic: record.topic,
|
|
316
|
+
partition: record.partition,
|
|
317
|
+
offset: record.offset,
|
|
318
|
+
key: readKeyLeniently(record.key),
|
|
319
|
+
headers,
|
|
320
|
+
payload: parsePayload(record.value),
|
|
321
|
+
timestamp: record.timestamp
|
|
322
|
+
};
|
|
323
|
+
const type = resolveText(options.type, { record: plain, name: "type" });
|
|
324
|
+
if (type === void 0) {
|
|
325
|
+
throw invalid("Record has no event type: the type option found none");
|
|
326
|
+
}
|
|
327
|
+
assertValidType(type, "The event type");
|
|
328
|
+
return {
|
|
329
|
+
id: resolveText(options.id, { record: plain, name: "id" }) ?? `kafka:${record.topic}:${record.partition}:${record.offset}`,
|
|
330
|
+
type,
|
|
331
|
+
source: resolveText(options.source, { record: plain, name: "source" }) ?? record.topic,
|
|
332
|
+
key: options.key === void 0 ? plain.key : resolveText(options.key, { record: plain, name: "key" }) ?? null,
|
|
333
|
+
payload: plain.payload,
|
|
334
|
+
headers,
|
|
335
|
+
correlationId: null,
|
|
336
|
+
causationId: null,
|
|
337
|
+
occurredAt: resolveOccurredAt(options.occurredAt, { plain, record })
|
|
338
|
+
};
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
function completeDecodedEvent(decoded, record) {
|
|
342
|
+
if (typeof decoded !== "object" || decoded === null) {
|
|
343
|
+
throw invalid("The decoder must return an event object synchronously");
|
|
344
|
+
}
|
|
345
|
+
if ("then" in decoded) {
|
|
346
|
+
observeRejection(decoded);
|
|
347
|
+
throw invalid("The decoder must return an event object synchronously");
|
|
348
|
+
}
|
|
349
|
+
const event = decoded;
|
|
350
|
+
return {
|
|
351
|
+
id: event.id,
|
|
352
|
+
type: event.type,
|
|
353
|
+
source: event.source ?? record.topic,
|
|
354
|
+
key: event.key === void 0 ? readRecordKey(record.key) : event.key,
|
|
355
|
+
payload: event.payload,
|
|
356
|
+
headers: event.headers ?? {},
|
|
357
|
+
correlationId: event.correlationId ?? null,
|
|
358
|
+
causationId: event.causationId ?? null,
|
|
359
|
+
occurredAt: event.occurredAt ?? requireRecordTimestamp(record)
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
function extensionAttributes(body) {
|
|
363
|
+
return Object.fromEntries(
|
|
364
|
+
Object.entries(body).filter(
|
|
365
|
+
(entry) => !STRUCTURED_ATTRIBUTES.has(entry[0]) && typeof entry[1] === "string"
|
|
366
|
+
)
|
|
367
|
+
);
|
|
368
|
+
}
|
|
369
|
+
function requireRecordTimestamp(record) {
|
|
370
|
+
const time = readRecordTimestamp(record.timestamp);
|
|
371
|
+
if (time === void 0) {
|
|
372
|
+
throw invalid("The event has no time and the record has no usable timestamp");
|
|
373
|
+
}
|
|
374
|
+
return time;
|
|
375
|
+
}
|
|
376
|
+
function readHeadersLeniently(incoming) {
|
|
377
|
+
const entries = [];
|
|
378
|
+
for (const [name, raw] of Object.entries(incoming ?? {})) {
|
|
379
|
+
const value = Array.isArray(raw) ? raw.at(-1) : raw;
|
|
380
|
+
const text = value === void 0 ? void 0 : decodeUtf8OrUndefined(value);
|
|
381
|
+
if (text !== void 0) {
|
|
382
|
+
entries.push([name, text]);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
return Object.fromEntries(entries);
|
|
386
|
+
}
|
|
387
|
+
function decodeUtf8OrUndefined(value) {
|
|
388
|
+
if (typeof value === "string") {
|
|
389
|
+
return value;
|
|
390
|
+
}
|
|
391
|
+
try {
|
|
392
|
+
return exactUtf8.decode(value);
|
|
393
|
+
} catch {
|
|
394
|
+
return void 0;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
function readKeyLeniently(key) {
|
|
398
|
+
if (key === null) {
|
|
399
|
+
return null;
|
|
400
|
+
}
|
|
401
|
+
try {
|
|
402
|
+
return readRecordKey(key);
|
|
403
|
+
} catch {
|
|
404
|
+
return Buffer.from(key).toString("base64");
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
function resolveText(option, { record, name }) {
|
|
408
|
+
const value = typeof option === "function" ? option(record) : option;
|
|
409
|
+
if (value === void 0 || value === null) {
|
|
410
|
+
return void 0;
|
|
411
|
+
}
|
|
412
|
+
if (typeof value !== "string") {
|
|
413
|
+
observeRejection(value);
|
|
414
|
+
throw invalid(`The ${name} option returned ${typeof value}, expected a string`);
|
|
415
|
+
}
|
|
416
|
+
return value.length === 0 ? void 0 : value;
|
|
417
|
+
}
|
|
418
|
+
function resolveOccurredAt(option, { plain, record }) {
|
|
419
|
+
const value = option?.(plain);
|
|
420
|
+
if (value === void 0) {
|
|
421
|
+
return requireRecordTimestamp(record);
|
|
422
|
+
}
|
|
423
|
+
if (!(value instanceof Date) || Number.isNaN(value.getTime())) {
|
|
424
|
+
observeRejection(value);
|
|
425
|
+
throw invalid("The occurredAt option returned no valid Date");
|
|
426
|
+
}
|
|
427
|
+
return value;
|
|
428
|
+
}
|
|
429
|
+
function observeRejection(value) {
|
|
430
|
+
if (typeof value === "object" && value !== null && "then" in value) {
|
|
431
|
+
Promise.resolve(value).catch(() => void 0);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// src/codecs.ts
|
|
436
|
+
var cloudEventsBinary = Object.freeze({
|
|
437
|
+
encode: encodeEnvelope,
|
|
438
|
+
decode: decodeEnvelope
|
|
439
|
+
});
|
|
440
|
+
var cloudEventsStructured = Object.freeze({
|
|
441
|
+
encode: encodeStructuredEnvelope,
|
|
442
|
+
decode: decodeCloudEvent
|
|
443
|
+
});
|
|
444
|
+
var RECORD_POSITION = "record-position";
|
|
445
|
+
var EVENT_ID_HEADER = "relaybus-event-id";
|
|
446
|
+
var DEFAULT_CONTENT_TYPE = "application/json";
|
|
447
|
+
function jsonEnvelope(mapping) {
|
|
448
|
+
const format = createJsonEnvelopeFormat(mapping, {
|
|
449
|
+
transportIds: [RECORD_POSITION],
|
|
450
|
+
failures: { invalid, unencodable: (reason) => new UnencodableEventError(reason) }
|
|
451
|
+
});
|
|
452
|
+
const contentType = mapping.contentType ?? DEFAULT_CONTENT_TYPE;
|
|
453
|
+
const stampsTime = mapping.occurredAt === void 0;
|
|
454
|
+
const ownHeaders = (envelope) => mapping.id === RECORD_POSITION ? { [CONTENT_TYPE]: contentType, [EVENT_ID_HEADER]: envelope.id } : { [CONTENT_TYPE]: contentType };
|
|
455
|
+
return {
|
|
456
|
+
encode: (envelope) => {
|
|
457
|
+
assertRepresentable(envelope);
|
|
458
|
+
const { body, headers } = format.write(envelope);
|
|
459
|
+
const taken = [CONTENT_TYPE, EVENT_ID_HEADER].find((name) => Object.hasOwn(headers, name));
|
|
460
|
+
if (taken !== void 0) {
|
|
461
|
+
throw new UnencodableEventError(
|
|
462
|
+
`Event ${envelope.id} has a header '${taken}', which the codec writes itself`
|
|
463
|
+
);
|
|
464
|
+
}
|
|
465
|
+
return {
|
|
466
|
+
key: envelope.key,
|
|
467
|
+
value: serializeJson(envelope, body),
|
|
468
|
+
headers: { ...headers, ...ownHeaders(envelope) },
|
|
469
|
+
...stampsTime ? { timestamp: recordTimestamp(envelope) } : {}
|
|
470
|
+
};
|
|
471
|
+
},
|
|
472
|
+
decode: (record) => {
|
|
473
|
+
const { [EVENT_ID_HEADER]: eventId, ...headers } = withoutContentType(
|
|
474
|
+
readHeadersLeniently(record.headers)
|
|
475
|
+
);
|
|
476
|
+
const fields = format.read({ body: parsePayload(record.value), headers });
|
|
477
|
+
return {
|
|
478
|
+
...fields,
|
|
479
|
+
id: fields.id ?? (eventId === void 0 || eventId === "" ? `kafka:${record.topic}:${record.partition}:${record.offset}` : eventId)
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
function withoutContentType(headers) {
|
|
485
|
+
return Object.fromEntries(Object.entries(headers).filter(([name]) => name !== CONTENT_TYPE));
|
|
486
|
+
}
|
|
487
|
+
function assertRepresentable(envelope) {
|
|
488
|
+
const kind = typeof envelope.payload;
|
|
489
|
+
if (kind === "undefined" || kind === "function" || kind === "symbol") {
|
|
490
|
+
throw new UnencodableEventError(
|
|
491
|
+
`Event ${envelope.id} has a payload that JSON cannot represent (undefined, a function or a symbol)`
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
function recordTimestamp(envelope) {
|
|
496
|
+
formatTime(envelope);
|
|
497
|
+
return String(envelope.occurredAt.getTime());
|
|
498
|
+
}
|
|
215
499
|
var DEFAULTS = {
|
|
216
500
|
name: "kafka",
|
|
217
501
|
maxInFlightRequests: 1,
|
|
@@ -226,6 +510,7 @@ var DEFAULTS = {
|
|
|
226
510
|
};
|
|
227
511
|
var KafkaTransport = class {
|
|
228
512
|
name;
|
|
513
|
+
system = "kafka";
|
|
229
514
|
#kafka;
|
|
230
515
|
#groupId;
|
|
231
516
|
#fromBeginning;
|
|
@@ -233,6 +518,7 @@ var KafkaTransport = class {
|
|
|
233
518
|
#producerConfig;
|
|
234
519
|
#consumerConfig;
|
|
235
520
|
#invalidPolicy;
|
|
521
|
+
#codec;
|
|
236
522
|
#logger;
|
|
237
523
|
#settings;
|
|
238
524
|
#sleepers = /* @__PURE__ */ new Set();
|
|
@@ -251,15 +537,13 @@ var KafkaTransport = class {
|
|
|
251
537
|
this.#kafka = options.kafka;
|
|
252
538
|
this.#groupId = options.groupId;
|
|
253
539
|
this.#fromBeginning = options.fromBeginning ?? true;
|
|
254
|
-
this.#maxInFlightRequests =
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
`maxInFlightRequests must be an integer from 1 to 5, received ${this.#maxInFlightRequests}`
|
|
258
|
-
);
|
|
259
|
-
}
|
|
540
|
+
this.#maxInFlightRequests = assertMaxInFlightRequests(
|
|
541
|
+
options.maxInFlightRequests ?? DEFAULTS.maxInFlightRequests
|
|
542
|
+
);
|
|
260
543
|
this.#producerConfig = options.producerConfig ?? {};
|
|
261
544
|
this.#consumerConfig = options.consumerConfig ?? {};
|
|
262
545
|
this.#invalidPolicy = options.onInvalidMessage ?? "dead-letter";
|
|
546
|
+
this.#codec = resolveCodec(options);
|
|
263
547
|
this.#logger = options.logger ?? console;
|
|
264
548
|
this.#settings = resolveSettings(options);
|
|
265
549
|
}
|
|
@@ -289,11 +573,10 @@ var KafkaTransport = class {
|
|
|
289
573
|
}
|
|
290
574
|
async publish(envelope, destination) {
|
|
291
575
|
const producer = await this.#connectedProducer();
|
|
292
|
-
const { key, value, headers } = encodeEnvelope(envelope);
|
|
293
576
|
await this.#send(producer, {
|
|
294
577
|
topic: destination,
|
|
295
578
|
acks: -1,
|
|
296
|
-
messages: [
|
|
579
|
+
messages: [this.#codec.encode(envelope)]
|
|
297
580
|
});
|
|
298
581
|
}
|
|
299
582
|
/**
|
|
@@ -303,13 +586,14 @@ var KafkaTransport = class {
|
|
|
303
586
|
* @throws ConfigurationError when `groupId` is missing, `destinations` is empty
|
|
304
587
|
* or the transport already subscribed (create one transport per subscription).
|
|
305
588
|
*/
|
|
306
|
-
|
|
589
|
+
// eslint-disable-next-line relaybus/max-two-params -- the Transport.subscribe signature
|
|
590
|
+
async subscribe(destinations, onMessage, hooks) {
|
|
307
591
|
this.#assertCanSubscribe(destinations);
|
|
308
592
|
this.#state = "subscribed";
|
|
309
593
|
const consumer = this.#kafka.consumer({ ...this.#consumerConfig, groupId: this.#groupId });
|
|
310
594
|
this.#consumer = consumer;
|
|
311
595
|
this.#observeConsumer(consumer);
|
|
312
|
-
const start = () => this.#startConsumer(consumer, [...destinations], onMessage);
|
|
596
|
+
const start = () => this.#startConsumer(consumer, { destinations: [...destinations], onMessage, hooks });
|
|
313
597
|
this.#startConsuming = start;
|
|
314
598
|
try {
|
|
315
599
|
await start();
|
|
@@ -325,12 +609,18 @@ var KafkaTransport = class {
|
|
|
325
609
|
`[KafkaTransport](subscribe) Consuming | transport=${this.name} group=${this.#groupId} topics=${destinations.join(",")}`
|
|
326
610
|
);
|
|
327
611
|
}
|
|
328
|
-
async #startConsumer(consumer,
|
|
612
|
+
async #startConsumer(consumer, {
|
|
613
|
+
destinations,
|
|
614
|
+
onMessage,
|
|
615
|
+
hooks
|
|
616
|
+
}) {
|
|
329
617
|
await consumer.connect();
|
|
330
618
|
await consumer.subscribe({ topics: destinations, fromBeginning: this.#fromBeginning });
|
|
331
619
|
const joined = this.#waitForGroupJoin(consumer);
|
|
332
620
|
try {
|
|
333
|
-
await consumer.run({
|
|
621
|
+
await consumer.run({
|
|
622
|
+
eachMessage: (payload) => this.#handleMessage(payload, { onMessage, hooks })
|
|
623
|
+
});
|
|
334
624
|
await joined.promise;
|
|
335
625
|
} finally {
|
|
336
626
|
joined.dispose();
|
|
@@ -348,7 +638,7 @@ var KafkaTransport = class {
|
|
|
348
638
|
for (const wake of this.#sleepers) {
|
|
349
639
|
wake();
|
|
350
640
|
}
|
|
351
|
-
this.#joinWaiter?.reject(new
|
|
641
|
+
this.#joinWaiter?.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
|
|
352
642
|
await this.#restartLoop;
|
|
353
643
|
const producer = await pendingProducer?.catch(() => void 0);
|
|
354
644
|
const results = await Promise.allSettled([
|
|
@@ -362,7 +652,7 @@ var KafkaTransport = class {
|
|
|
362
652
|
}
|
|
363
653
|
#assertCanSubscribe(destinations) {
|
|
364
654
|
if (this.#state === "closed") {
|
|
365
|
-
throw new
|
|
655
|
+
throw new TransportClosedError(`Transport '${this.name}' is closed`);
|
|
366
656
|
}
|
|
367
657
|
if (this.#state === "subscribed") {
|
|
368
658
|
throw new ConfigurationError(
|
|
@@ -398,27 +688,30 @@ var KafkaTransport = class {
|
|
|
398
688
|
}
|
|
399
689
|
/** Sends with the publish timeout; the late outcome of a timed-out send is ignored. */
|
|
400
690
|
async #send(producer, record) {
|
|
401
|
-
let
|
|
691
|
+
let cancel;
|
|
402
692
|
const timeout = new Promise((_resolve, reject) => {
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
693
|
+
cancel = schedule(systemClock, {
|
|
694
|
+
run: () => {
|
|
695
|
+
reject(
|
|
696
|
+
new PublishTimeoutError(
|
|
697
|
+
`Transport '${this.name}' got no broker ack for topic '${record.topic}' within ${this.#settings.publishTimeoutMs}ms`
|
|
698
|
+
)
|
|
699
|
+
);
|
|
700
|
+
},
|
|
701
|
+
delayMs: this.#settings.publishTimeoutMs
|
|
702
|
+
});
|
|
410
703
|
});
|
|
411
704
|
const sending = producer.send(record);
|
|
412
705
|
sending.catch(() => void 0);
|
|
413
706
|
try {
|
|
414
707
|
await Promise.race([sending, timeout]);
|
|
415
708
|
} finally {
|
|
416
|
-
|
|
709
|
+
cancel?.();
|
|
417
710
|
}
|
|
418
711
|
}
|
|
419
712
|
#connectedProducer() {
|
|
420
713
|
if (this.#state === "closed") {
|
|
421
|
-
return Promise.reject(new
|
|
714
|
+
return Promise.reject(new TransportClosedError(`Transport '${this.name}' is closed`));
|
|
422
715
|
}
|
|
423
716
|
this.#producer ??= this.#openProducer();
|
|
424
717
|
return this.#producer;
|
|
@@ -438,40 +731,63 @@ var KafkaTransport = class {
|
|
|
438
731
|
this.#producerConnected = this.#state !== "closed";
|
|
439
732
|
return producer;
|
|
440
733
|
}
|
|
441
|
-
async #handleMessage(payload,
|
|
734
|
+
async #handleMessage(payload, {
|
|
735
|
+
onMessage,
|
|
736
|
+
hooks
|
|
737
|
+
}) {
|
|
442
738
|
let envelope;
|
|
443
739
|
try {
|
|
444
|
-
envelope =
|
|
740
|
+
envelope = this.#decodeRecord(payload);
|
|
445
741
|
} catch (error) {
|
|
446
|
-
if (!(error instanceof InvalidKafkaMessageError)) {
|
|
742
|
+
if (this.#codec === cloudEventsBinary && !(error instanceof InvalidKafkaMessageError)) {
|
|
447
743
|
throw toHandlerFailure(error);
|
|
448
744
|
}
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
"handleInvalid",
|
|
452
|
-
() => this.#handleInvalid(payload, error
|
|
453
|
-
);
|
|
745
|
+
hooks?.onInvalidMessage({ destination: payload.topic, error });
|
|
746
|
+
await this.#withRetries(payload, {
|
|
747
|
+
method: "handleInvalid",
|
|
748
|
+
operation: () => this.#handleInvalid(payload, error)
|
|
749
|
+
});
|
|
454
750
|
return;
|
|
455
751
|
}
|
|
456
|
-
await this.#withRetries(payload,
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
752
|
+
await this.#withRetries(payload, {
|
|
753
|
+
method: "deliver",
|
|
754
|
+
operation: async () => {
|
|
755
|
+
try {
|
|
756
|
+
await onMessage(envelope, { destination: payload.topic });
|
|
757
|
+
} catch (error) {
|
|
758
|
+
if (!(error instanceof NonRetryableError)) {
|
|
759
|
+
throw error;
|
|
760
|
+
}
|
|
761
|
+
await this.#handleInvalid(payload, error);
|
|
462
762
|
}
|
|
463
|
-
await this.#handleInvalid(payload, error.message);
|
|
464
763
|
}
|
|
465
764
|
});
|
|
466
765
|
}
|
|
467
|
-
|
|
766
|
+
#decodeRecord({ topic, partition, message }) {
|
|
767
|
+
if (this.#codec === cloudEventsBinary) {
|
|
768
|
+
return decodeEnvelope(message);
|
|
769
|
+
}
|
|
770
|
+
const record = {
|
|
771
|
+
topic,
|
|
772
|
+
partition,
|
|
773
|
+
offset: message.offset,
|
|
774
|
+
key: message.key,
|
|
775
|
+
value: message.value,
|
|
776
|
+
headers: message.headers,
|
|
777
|
+
timestamp: message.timestamp
|
|
778
|
+
};
|
|
779
|
+
return completeDecodedEvent(this.#codec.decode(record), record);
|
|
780
|
+
}
|
|
781
|
+
async #handleInvalid(payload, error) {
|
|
468
782
|
const { topic, partition, message } = payload;
|
|
783
|
+
const { message: reason, name, code } = toErrorInfo(error);
|
|
469
784
|
const location = `topic=${topic} partition=${partition} offset=${message.offset}`;
|
|
785
|
+
const errorFields = `errorName=${name}${code === void 0 ? "" : ` code=${code}`}`;
|
|
470
786
|
const policy = this.#invalidPolicy;
|
|
471
787
|
const deadLetterTopic = this.#deadLetterTopicFor(topic);
|
|
472
788
|
if (policy === "skip") {
|
|
473
789
|
this.#logger.error(
|
|
474
|
-
`[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason}`
|
|
790
|
+
`[KafkaTransport](handleInvalid) Skipping malformed message | ${location} reason=${reason} ${errorFields}`
|
|
475
791
|
);
|
|
476
792
|
} else if (typeof policy === "function") {
|
|
477
793
|
await policy({
|
|
@@ -483,6 +799,19 @@ var KafkaTransport = class {
|
|
|
483
799
|
value: message.value
|
|
484
800
|
});
|
|
485
801
|
} else {
|
|
802
|
+
const headers = {
|
|
803
|
+
...toOutgoingHeaders(message.headers),
|
|
804
|
+
"relaybus-invalid-reason": reason,
|
|
805
|
+
"relaybus-source-topic": topic,
|
|
806
|
+
"relaybus-source-partition": String(partition),
|
|
807
|
+
"relaybus-source-offset": message.offset,
|
|
808
|
+
"relaybus-error-name": name
|
|
809
|
+
};
|
|
810
|
+
if (code === void 0) {
|
|
811
|
+
delete headers["relaybus-error-code"];
|
|
812
|
+
} else {
|
|
813
|
+
headers["relaybus-error-code"] = code;
|
|
814
|
+
}
|
|
486
815
|
const producer = await this.#connectedProducer();
|
|
487
816
|
await this.#send(producer, {
|
|
488
817
|
topic: deadLetterTopic,
|
|
@@ -491,18 +820,12 @@ var KafkaTransport = class {
|
|
|
491
820
|
{
|
|
492
821
|
key: message.key === null ? null : Buffer.from(message.key),
|
|
493
822
|
value: message.value === null ? null : Buffer.from(message.value),
|
|
494
|
-
headers
|
|
495
|
-
...toOutgoingHeaders(message.headers),
|
|
496
|
-
"relaybus-invalid-reason": reason,
|
|
497
|
-
"relaybus-source-topic": topic,
|
|
498
|
-
"relaybus-source-partition": String(partition),
|
|
499
|
-
"relaybus-source-offset": message.offset
|
|
500
|
-
}
|
|
823
|
+
headers
|
|
501
824
|
}
|
|
502
825
|
]
|
|
503
826
|
});
|
|
504
827
|
this.#logger.error(
|
|
505
|
-
`[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason}`
|
|
828
|
+
`[KafkaTransport](handleInvalid) Dead-lettered malformed message | ${location} deadLetterTopic=${deadLetterTopic} reason=${reason} ${errorFields}`
|
|
506
829
|
);
|
|
507
830
|
}
|
|
508
831
|
}
|
|
@@ -511,7 +834,7 @@ var KafkaTransport = class {
|
|
|
511
834
|
* last error is rethrown so the client leaves the offset uncommitted and
|
|
512
835
|
* redelivers the record; heartbeats between attempts keep the session alive.
|
|
513
836
|
*/
|
|
514
|
-
async #withRetries(payload, method, operation) {
|
|
837
|
+
async #withRetries(payload, { method, operation }) {
|
|
515
838
|
const { maxAttempts, initialBackoffMs, maxBackoffMs } = this.#settings;
|
|
516
839
|
const location = `topic=${payload.topic} partition=${payload.partition} offset=${payload.message.offset}`;
|
|
517
840
|
const partitionKey = `${payload.topic}:${payload.partition}`;
|
|
@@ -525,12 +848,12 @@ var KafkaTransport = class {
|
|
|
525
848
|
if (attempt >= maxAttempts || this.#state === "closed") {
|
|
526
849
|
this.#stalledPartitions.add(partitionKey);
|
|
527
850
|
this.#logger.error(
|
|
528
|
-
`[KafkaTransport](
|
|
851
|
+
`[KafkaTransport](withRetries) Giving up for now, the record stays uncommitted and will be redelivered | operation=${method} ${location} attempts=${attempt} error=${reason}`
|
|
529
852
|
);
|
|
530
853
|
throw toHandlerFailure(error);
|
|
531
854
|
}
|
|
532
855
|
this.#logger.warn(
|
|
533
|
-
`[KafkaTransport](
|
|
856
|
+
`[KafkaTransport](withRetries) Attempt failed, retrying | operation=${method} ${location} attempt=${attempt} error=${reason}`
|
|
534
857
|
);
|
|
535
858
|
await this.#sleepWithHeartbeats(
|
|
536
859
|
Math.min(initialBackoffMs * 2 ** (attempt - 1), maxBackoffMs),
|
|
@@ -552,11 +875,11 @@ var KafkaTransport = class {
|
|
|
552
875
|
#sleep(milliseconds) {
|
|
553
876
|
return new Promise((resolve) => {
|
|
554
877
|
const wake = () => {
|
|
555
|
-
|
|
878
|
+
cancel();
|
|
556
879
|
this.#sleepers.delete(wake);
|
|
557
880
|
resolve();
|
|
558
881
|
};
|
|
559
|
-
const
|
|
882
|
+
const cancel = schedule(systemClock, { run: wake, delayMs: milliseconds });
|
|
560
883
|
this.#sleepers.add(wake);
|
|
561
884
|
});
|
|
562
885
|
}
|
|
@@ -638,42 +961,53 @@ var KafkaTransport = class {
|
|
|
638
961
|
if (!this.#tracksMembership(consumer)) {
|
|
639
962
|
return { promise: Promise.resolve(), dispose: () => void 0 };
|
|
640
963
|
}
|
|
641
|
-
let
|
|
964
|
+
let cancel;
|
|
642
965
|
const promise = new Promise((resolve, reject) => {
|
|
643
966
|
this.#joinWaiter = { resolve, reject };
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
967
|
+
cancel = schedule(systemClock, {
|
|
968
|
+
run: () => {
|
|
969
|
+
reject(
|
|
970
|
+
new GroupJoinTimeoutError(
|
|
971
|
+
`Consumer of transport '${this.name}' did not join group '${this.#groupId}' within ${this.#settings.startupTimeoutMs}ms`
|
|
972
|
+
)
|
|
973
|
+
);
|
|
974
|
+
},
|
|
975
|
+
delayMs: this.#settings.startupTimeoutMs
|
|
976
|
+
});
|
|
651
977
|
});
|
|
652
978
|
promise.catch(() => void 0);
|
|
653
979
|
return {
|
|
654
980
|
promise,
|
|
655
981
|
dispose: () => {
|
|
656
|
-
|
|
982
|
+
cancel?.();
|
|
657
983
|
this.#joinWaiter = void 0;
|
|
658
984
|
}
|
|
659
985
|
};
|
|
660
986
|
}
|
|
661
987
|
};
|
|
662
|
-
function
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
throw new ConfigurationError(
|
|
666
|
-
`processing.maxAttempts must be a positive integer, received ${maxAttempts}`
|
|
667
|
-
);
|
|
988
|
+
function resolveCodec({ codec, decode }) {
|
|
989
|
+
if (decode === void 0) {
|
|
990
|
+
return codec ?? cloudEventsBinary;
|
|
668
991
|
}
|
|
992
|
+
if (codec !== void 0) {
|
|
993
|
+
throw new ConfigurationError("Give 'codec' or the deprecated 'decode', not both");
|
|
994
|
+
}
|
|
995
|
+
return { encode: encodeEnvelope, decode };
|
|
996
|
+
}
|
|
997
|
+
function resolveSettings(options) {
|
|
998
|
+
const processing = options.processing ?? {};
|
|
999
|
+
const maxAttempts = assertPositiveInteger(
|
|
1000
|
+
processing.maxAttempts ?? DEFAULTS.maxAttempts,
|
|
1001
|
+
"processing.maxAttempts"
|
|
1002
|
+
);
|
|
669
1003
|
return {
|
|
670
1004
|
maxAttempts,
|
|
671
1005
|
initialBackoffMs: toMilliseconds(
|
|
672
|
-
|
|
1006
|
+
processing.initialBackoff ?? DEFAULTS.initialBackoff,
|
|
673
1007
|
"processing.initialBackoff"
|
|
674
1008
|
),
|
|
675
1009
|
maxBackoffMs: toMilliseconds(
|
|
676
|
-
|
|
1010
|
+
processing.maxBackoff ?? DEFAULTS.maxBackoff,
|
|
677
1011
|
"processing.maxBackoff"
|
|
678
1012
|
),
|
|
679
1013
|
startupTimeoutMs: toMilliseconds(
|
|
@@ -687,25 +1021,39 @@ function resolveSettings(options) {
|
|
|
687
1021
|
heartbeatIntervalMs: Math.max(
|
|
688
1022
|
1,
|
|
689
1023
|
toMilliseconds(
|
|
690
|
-
|
|
1024
|
+
processing.heartbeatInterval ?? DEFAULTS.heartbeatInterval,
|
|
691
1025
|
"processing.heartbeatInterval"
|
|
692
1026
|
)
|
|
693
1027
|
),
|
|
1028
|
+
...resolveRestartSettings(options.restart ?? {})
|
|
1029
|
+
};
|
|
1030
|
+
}
|
|
1031
|
+
function resolveRestartSettings(restart) {
|
|
1032
|
+
return {
|
|
694
1033
|
restartInitialBackoffMs: toMilliseconds(
|
|
695
|
-
|
|
1034
|
+
restart.initialBackoff ?? DEFAULTS.restartInitialBackoff,
|
|
696
1035
|
"restart.initialBackoff"
|
|
697
1036
|
),
|
|
698
1037
|
restartMaxBackoffMs: toMilliseconds(
|
|
699
|
-
|
|
1038
|
+
restart.maxBackoff ?? DEFAULTS.restartMaxBackoff,
|
|
700
1039
|
"restart.maxBackoff"
|
|
701
1040
|
)
|
|
702
1041
|
};
|
|
703
1042
|
}
|
|
1043
|
+
var MAXIMUM_IN_FLIGHT_REQUESTS = 5;
|
|
1044
|
+
function assertMaxInFlightRequests(value) {
|
|
1045
|
+
if (!Number.isInteger(value) || value < 1 || value > MAXIMUM_IN_FLIGHT_REQUESTS) {
|
|
1046
|
+
throw new ConfigurationError(
|
|
1047
|
+
`maxInFlightRequests must be an integer from 1 to 5, received ${value}`
|
|
1048
|
+
);
|
|
1049
|
+
}
|
|
1050
|
+
return value;
|
|
1051
|
+
}
|
|
704
1052
|
function toHandlerFailure(error) {
|
|
705
1053
|
const { name, message } = toErrorInfo(error);
|
|
706
1054
|
return new HandlerFailedError(`${name}: ${message}`);
|
|
707
1055
|
}
|
|
708
1056
|
|
|
709
|
-
export { HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, UnencodableEventError, decodeEnvelope, encodeEnvelope };
|
|
1057
|
+
export { GroupJoinTimeoutError, HandlerFailedError, InvalidKafkaMessageError, KafkaTransport, PublishTimeoutError, TransportClosedError, UnencodableEventError, cloudEventsBinary, cloudEventsStructured, decodeCloudEvent, decodeEnvelope, encodeEnvelope, jsonEnvelope, plainJsonDecoder };
|
|
710
1058
|
//# sourceMappingURL=index.js.map
|
|
711
1059
|
//# sourceMappingURL=index.js.map
|