@eventferry/kafka 3.3.1 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +184 -0
- package/dist/consume.cjs +115 -0
- package/dist/consume.cjs.map +1 -0
- package/dist/consume.d.cts +114 -0
- package/dist/consume.d.ts +114 -0
- package/dist/consume.js +88 -0
- package/dist/consume.js.map +1 -0
- package/dist/index.cjs +269 -7
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +218 -1
- package/dist/index.d.ts +218 -1
- package/dist/index.js +269 -7
- package/dist/index.js.map +1 -1
- package/package.json +6 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/consume.ts"],"sourcesContent":["/**\n * Consumer-side helpers — paired with the publisher's outbound surface.\n *\n * eventferry is a publisher-only library, but the messages it produces are\n * consumed somewhere downstream. This module normalizes the message shape\n * that both `kafkajs` and `@confluentinc/kafka-javascript` deliver to\n * consumer callbacks, decodes the payload, and extracts the W3C trace\n * context the publisher injected (see {@link KafkaTracer.inject}).\n *\n * Imported via subpath so consumer code paths don't pull in the producer:\n *\n * import { decode, extractTraceContext } from \"@eventferry/kafka/consume\";\n *\n * There is intentionally NO Kafka client here — bring your own consumer\n * (kafkajs's `Consumer`, librdkafka's, whatever) and call `decode()` /\n * `extractTraceContext()` on the message you receive.\n */\n\n/**\n * Raw incoming Kafka message — structural subset both kafkajs and confluent\n * (via the kafkaJS-compat layer) deliver. Fields are optional because\n * different consumer APIs surface different subsets.\n */\nexport interface IncomingKafkaMessage {\n key?: Buffer | string | null;\n value?: Buffer | string | null;\n headers?: IncomingHeaders;\n /** ISO ms string or numeric epoch ms — depends on the client. */\n timestamp?: string | number;\n /** Numeric or string per client. */\n offset?: string | number;\n partition?: number;\n}\n\n/** Headers as the underlying clients deliver them: bytes, strings, or undefined. */\nexport type IncomingHeaders = Record<string, Buffer | string | undefined>;\n\n/** Headers normalized to UTF-8 strings (the form most application code wants). */\nexport type DecodedHeaders = Record<string, string>;\n\n/** Payload decoder. Buffer in, decoded value out. */\nexport type Decoder<T> = (bytes: Buffer) => T;\n\n/** Decoded message wrapper — value plus normalized headers, key, metadata. */\nexport interface DecodedMessage<V = unknown> {\n key: string | null;\n value: V | null;\n headers: DecodedHeaders;\n /** Epoch ms when the broker stamped the record. */\n timestamp?: number;\n /** Stringified offset (Kafka offsets exceed 2^53 — strings stay safe). */\n offset?: string;\n partition?: number;\n}\n\nexport interface DecodeOptions<V> {\n /**\n * Decoder for the payload bytes. Built-ins:\n *\n * - `\"json\"` (default) — `JSON.parse(value.toString(\"utf8\"))`. Empty\n * value returns `null` (matches Kafka tombstones on compacted topics).\n * - `\"utf8\"` — raw text. Returns the string as-is.\n * - `\"none\"` — returns the raw `Buffer` unchanged.\n *\n * Or pass your own `(bytes: Buffer) => V` for Avro / Protobuf / MessagePack.\n */\n decoder?: \"json\" | \"utf8\" | \"none\" | Decoder<V>;\n}\n\n/**\n * Normalize headers to a plain string→string map. Buffers are read as UTF-8;\n * `undefined` entries are dropped (consumers occasionally surface absent\n * headers as `undefined` values).\n */\nexport function decodeHeaders(raw?: IncomingHeaders): DecodedHeaders {\n if (!raw) return {};\n const out: DecodedHeaders = {};\n for (const [k, v] of Object.entries(raw)) {\n if (v === undefined || v === null) continue;\n out[k] = Buffer.isBuffer(v) ? v.toString(\"utf8\") : v;\n }\n return out;\n}\n\n/**\n * Decode a Kafka message: normalize the key + headers, decode the value\n * with the chosen decoder, and surface the broker metadata.\n *\n * Tombstones (null/empty value) come back with `value: null` regardless of\n * the decoder — compaction-friendly.\n *\n * @throws when `decoder: \"json\"` (the default) and the payload is non-empty\n * but not valid JSON. Catch the error and decide whether to DLQ the\n * record or skip it — eventferry does not assume.\n */\nexport function decode<V = unknown>(\n msg: IncomingKafkaMessage,\n opts: DecodeOptions<V> = {},\n): DecodedMessage<V> {\n const headers = decodeHeaders(msg.headers);\n const key = normalizeKey(msg.key);\n const value = decodeValue<V>(msg.value, opts.decoder ?? \"json\");\n const timestamp =\n msg.timestamp !== undefined ? Number(msg.timestamp) : undefined;\n const offset = msg.offset !== undefined ? String(msg.offset) : undefined;\n return {\n key,\n value,\n headers,\n timestamp,\n offset,\n partition: msg.partition,\n };\n}\n\nfunction normalizeKey(key?: Buffer | string | null): string | null {\n if (key === null || key === undefined) return null;\n return Buffer.isBuffer(key) ? key.toString(\"utf8\") : key;\n}\n\nfunction decodeValue<V>(\n value: Buffer | string | null | undefined,\n decoder: DecodeOptions<V>[\"decoder\"],\n): V | null {\n if (value === null || value === undefined) return null;\n const buf = Buffer.isBuffer(value) ? value : Buffer.from(value);\n // Tombstones: an empty buffer is a kafka \"delete me\" on compacted\n // topics. Surface as null for every decoder — applications usually\n // want the same null-handling for both.\n if (buf.length === 0) return null;\n if (typeof decoder === \"function\") return decoder(buf);\n switch (decoder) {\n case \"utf8\":\n return buf.toString(\"utf8\") as unknown as V;\n case \"none\":\n return buf as unknown as V;\n case \"json\":\n case undefined:\n default: {\n const text = buf.toString(\"utf8\");\n try {\n return JSON.parse(text) as V;\n } catch (err) {\n throw new Error(\n `decode: JSON.parse failed on message value: ${(err as Error).message}`,\n );\n }\n }\n }\n}\n\n/**\n * W3C Trace Context extracted from message headers.\n *\n * - `traceparent`: full header value, format `version-traceId-spanId-flags`.\n * - `tracestate`: optional vendor-specific state (W3C `tracestate` header).\n * - `traceId`: 32 hex chars, parsed from `traceparent`.\n * - `spanId`: 16 hex chars (the PARENT span id from the producer).\n * - `sampled`: parsed from the `traceparent` flags (bit 0 = sampled).\n *\n * Returns `null` when no `traceparent` header is present or the value\n * fails W3C validation.\n *\n * Spec: https://www.w3.org/TR/trace-context/\n */\nexport interface TraceContext {\n traceparent: string;\n tracestate?: string;\n traceId: string;\n spanId: string;\n sampled: boolean;\n}\n\nconst TRACEPARENT_RE =\n /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/;\nconst INVALID_TRACE_ID = \"0\".repeat(32);\nconst INVALID_SPAN_ID = \"0\".repeat(16);\n\n/**\n * Extract the W3C trace context the publisher injected into headers.\n * Headers may be raw (Buffer values) or already-decoded (string values) —\n * both shapes work, so you can call this before OR after `decode()`.\n *\n * Validation follows the W3C spec strictly: invalid all-zero trace/span\n * IDs are rejected, version `ff` is rejected, malformed hex is rejected.\n * On any of these, the function returns `null` rather than throwing —\n * consumer code should fall back to starting a fresh trace.\n */\nexport function extractTraceContext(\n headers: IncomingHeaders | DecodedHeaders | undefined,\n): TraceContext | null {\n if (!headers) return null;\n const tp = readHeader(headers, \"traceparent\");\n if (!tp) return null;\n const match = TRACEPARENT_RE.exec(tp);\n if (!match) return null;\n const [, version, traceId, spanId, flags] = match as unknown as [\n string,\n string,\n string,\n string,\n string,\n ];\n // Spec §3.2.2.5: version \"ff\" is forbidden (reserved sentinel).\n if (version === \"ff\") return null;\n if (traceId === INVALID_TRACE_ID || spanId === INVALID_SPAN_ID) return null;\n const sampled = (parseInt(flags, 16) & 0x01) === 1;\n const ts = readHeader(headers, \"tracestate\");\n return {\n traceparent: tp,\n tracestate: ts && ts.length > 0 ? ts : undefined,\n traceId,\n spanId,\n sampled,\n };\n}\n\nfunction readHeader(\n headers: IncomingHeaders | DecodedHeaders,\n name: string,\n): string | undefined {\n const v = (headers as Record<string, Buffer | string | undefined>)[name];\n if (v === undefined || v === null) return undefined;\n if (typeof v === \"string\") return v;\n if (Buffer.isBuffer(v)) return v.toString(\"utf8\");\n return undefined;\n}\n"],"mappings":";AA0EO,SAAS,cAAc,KAAuC;AACnE,MAAI,CAAC,IAAK,QAAO,CAAC;AAClB,QAAM,MAAsB,CAAC;AAC7B,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,GAAG,GAAG;AACxC,QAAI,MAAM,UAAa,MAAM,KAAM;AACnC,QAAI,CAAC,IAAI,OAAO,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,IAAI;AAAA,EACrD;AACA,SAAO;AACT;AAaO,SAAS,OACd,KACA,OAAyB,CAAC,GACP;AACnB,QAAM,UAAU,cAAc,IAAI,OAAO;AACzC,QAAM,MAAM,aAAa,IAAI,GAAG;AAChC,QAAM,QAAQ,YAAe,IAAI,OAAO,KAAK,WAAW,MAAM;AAC9D,QAAM,YACJ,IAAI,cAAc,SAAY,OAAO,IAAI,SAAS,IAAI;AACxD,QAAM,SAAS,IAAI,WAAW,SAAY,OAAO,IAAI,MAAM,IAAI;AAC/D,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,IAAI;AAAA,EACjB;AACF;AAEA,SAAS,aAAa,KAA6C;AACjE,MAAI,QAAQ,QAAQ,QAAQ,OAAW,QAAO;AAC9C,SAAO,OAAO,SAAS,GAAG,IAAI,IAAI,SAAS,MAAM,IAAI;AACvD;AAEA,SAAS,YACP,OACA,SACU;AACV,MAAI,UAAU,QAAQ,UAAU,OAAW,QAAO;AAClD,QAAM,MAAM,OAAO,SAAS,KAAK,IAAI,QAAQ,OAAO,KAAK,KAAK;AAI9D,MAAI,IAAI,WAAW,EAAG,QAAO;AAC7B,MAAI,OAAO,YAAY,WAAY,QAAO,QAAQ,GAAG;AACrD,UAAQ,SAAS;AAAA,IACf,KAAK;AACH,aAAO,IAAI,SAAS,MAAM;AAAA,IAC5B,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AAAA,IACL,SAAS;AACP,YAAM,OAAO,IAAI,SAAS,MAAM;AAChC,UAAI;AACF,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB,SAAS,KAAK;AACZ,cAAM,IAAI;AAAA,UACR,+CAAgD,IAAc,OAAO;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAwBA,IAAM,iBACJ;AACF,IAAM,mBAAmB,IAAI,OAAO,EAAE;AACtC,IAAM,kBAAkB,IAAI,OAAO,EAAE;AAY9B,SAAS,oBACd,SACqB;AACrB,MAAI,CAAC,QAAS,QAAO;AACrB,QAAM,KAAK,WAAW,SAAS,aAAa;AAC5C,MAAI,CAAC,GAAI,QAAO;AAChB,QAAM,QAAQ,eAAe,KAAK,EAAE;AACpC,MAAI,CAAC,MAAO,QAAO;AACnB,QAAM,CAAC,EAAE,SAAS,SAAS,QAAQ,KAAK,IAAI;AAQ5C,MAAI,YAAY,KAAM,QAAO;AAC7B,MAAI,YAAY,oBAAoB,WAAW,gBAAiB,QAAO;AACvE,QAAM,WAAW,SAAS,OAAO,EAAE,IAAI,OAAU;AACjD,QAAM,KAAK,WAAW,SAAS,YAAY;AAC3C,SAAO;AAAA,IACL,aAAa;AAAA,IACb,YAAY,MAAM,GAAG,SAAS,IAAI,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,WACP,SACA,MACoB;AACpB,QAAM,IAAK,QAAwD,IAAI;AACvE,MAAI,MAAM,UAAa,MAAM,KAAM,QAAO;AAC1C,MAAI,OAAO,MAAM,SAAU,QAAO;AAClC,MAAI,OAAO,SAAS,CAAC,EAAG,QAAO,EAAE,SAAS,MAAM;AAChD,SAAO;AACT;","names":[]}
|
package/dist/index.cjs
CHANGED
|
@@ -148,7 +148,10 @@ var UNSUPPORTED_BY_KAFKAJS = [
|
|
|
148
148
|
"lingerMs",
|
|
149
149
|
"batchSize",
|
|
150
150
|
"deliveryTimeoutMs",
|
|
151
|
-
"maxRequestSize"
|
|
151
|
+
"maxRequestSize",
|
|
152
|
+
// Confluent-only escape hatches; ignored on kafkajs.
|
|
153
|
+
"compressionLevel",
|
|
154
|
+
"rawProducerConfig"
|
|
152
155
|
];
|
|
153
156
|
var KafkaJsDriver = class {
|
|
154
157
|
transactional;
|
|
@@ -187,13 +190,21 @@ var KafkaJsDriver = class {
|
|
|
187
190
|
// the provider's returned token (other fields are ignored).
|
|
188
191
|
sasl: this.opts.sasl
|
|
189
192
|
});
|
|
190
|
-
|
|
191
|
-
|
|
193
|
+
return kafka.producer(await this.buildProducerOptions(mod.Partitioners));
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Compute the options object passed to `kafka.producer({...})`. Exposed
|
|
197
|
+
* as a test seam so power-user escape hatches (customPartitioner,
|
|
198
|
+
* rawKafkaJsProducerConfig) can be asserted without a live broker.
|
|
199
|
+
*/
|
|
200
|
+
async buildProducerOptions(partitioners) {
|
|
201
|
+
const createPartitioner = this.opts.customPartitioner ?? resolveCreatePartitioner(
|
|
202
|
+
partitioners,
|
|
192
203
|
this.opts.partitioner,
|
|
193
204
|
this.transactional
|
|
194
205
|
);
|
|
195
206
|
const resolvedTxId = this.transactional ? await resolveTransactionalId(this.opts.transactionalId) : void 0;
|
|
196
|
-
return
|
|
207
|
+
return {
|
|
197
208
|
idempotent: this.opts.idempotent ?? true,
|
|
198
209
|
// Idempotent / transactional producers cap maxInFlight at 5. When the
|
|
199
210
|
// user picks transactional we force 1 to keep strict ordering across
|
|
@@ -207,13 +218,32 @@ var KafkaJsDriver = class {
|
|
|
207
218
|
transactionTimeout: this.opts.transactionTimeoutMs,
|
|
208
219
|
// Setting any partitioner choice silences kafkajs's
|
|
209
220
|
// KafkaJSPartitionerNotSpecified warning.
|
|
210
|
-
createPartitioner
|
|
211
|
-
|
|
221
|
+
createPartitioner,
|
|
222
|
+
// Power-user escape hatch — merged LAST so raw keys win against the
|
|
223
|
+
// translated ones. That's the contract: anything you put here is
|
|
224
|
+
// final, even if it overrides idempotent/transactionalId/etc.
|
|
225
|
+
...this.opts.rawKafkaJsProducerConfig ?? {}
|
|
226
|
+
};
|
|
212
227
|
}
|
|
213
228
|
async disconnect() {
|
|
214
229
|
await this.producer?.disconnect();
|
|
215
230
|
this.producer = null;
|
|
216
231
|
}
|
|
232
|
+
/**
|
|
233
|
+
* Construct a kafkajs admin client wrapped in the eventferry-facing
|
|
234
|
+
* `KafkaDriverAdmin` shape. The publisher calls `.connect()` on the
|
|
235
|
+
* returned object before exposing it via `publisher.admin()`.
|
|
236
|
+
*/
|
|
237
|
+
async admin() {
|
|
238
|
+
const mod = await importKafkaJs();
|
|
239
|
+
const kafka = new mod.Kafka({
|
|
240
|
+
clientId: this.opts.clientId ?? "eventferry-admin",
|
|
241
|
+
brokers: this.opts.brokers,
|
|
242
|
+
ssl: this.opts.ssl,
|
|
243
|
+
sasl: this.opts.sasl
|
|
244
|
+
});
|
|
245
|
+
return new KafkaJsAdmin(kafka.admin());
|
|
246
|
+
}
|
|
217
247
|
async sendBatch(messages) {
|
|
218
248
|
if (!this.producer) throw new Error("KafkaJsDriver not connected");
|
|
219
249
|
const topicMessages = groupByTopic(messages, this.opts.compression);
|
|
@@ -302,6 +332,69 @@ function warnUnsupportedKafkajsOptions(opts) {
|
|
|
302
332
|
function _resetKafkajsWarnDedup() {
|
|
303
333
|
warnedKafkajsKeys.clear();
|
|
304
334
|
}
|
|
335
|
+
var KafkaJsAdmin = class {
|
|
336
|
+
constructor(client) {
|
|
337
|
+
this.client = client;
|
|
338
|
+
}
|
|
339
|
+
client;
|
|
340
|
+
async connect() {
|
|
341
|
+
await this.client.connect();
|
|
342
|
+
}
|
|
343
|
+
async close() {
|
|
344
|
+
await this.client.disconnect();
|
|
345
|
+
}
|
|
346
|
+
async listTopics() {
|
|
347
|
+
return await this.client.listTopics();
|
|
348
|
+
}
|
|
349
|
+
async describeTopics(topics) {
|
|
350
|
+
if (topics.length === 0) return [];
|
|
351
|
+
const all = new Set(await this.client.listTopics());
|
|
352
|
+
const existing = topics.filter((t) => all.has(t));
|
|
353
|
+
const missing = topics.filter((t) => !all.has(t));
|
|
354
|
+
const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
|
|
355
|
+
const byName = new Map(meta.topics.map((t) => [t.name, t]));
|
|
356
|
+
return topics.map((topic) => {
|
|
357
|
+
if (missing.includes(topic)) return { topic, partitions: [] };
|
|
358
|
+
const found = byName.get(topic);
|
|
359
|
+
if (!found) return { topic, partitions: [] };
|
|
360
|
+
return {
|
|
361
|
+
topic,
|
|
362
|
+
partitions: found.partitions.map((p) => ({
|
|
363
|
+
partitionId: p.partitionId,
|
|
364
|
+
leader: p.leader,
|
|
365
|
+
replicas: p.replicas,
|
|
366
|
+
isr: p.isr
|
|
367
|
+
}))
|
|
368
|
+
};
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
async createTopics(specs) {
|
|
372
|
+
if (specs.length === 0) return;
|
|
373
|
+
const topics = specs.map((s) => ({
|
|
374
|
+
topic: s.topic,
|
|
375
|
+
numPartitions: s.numPartitions,
|
|
376
|
+
replicationFactor: s.replicationFactor,
|
|
377
|
+
configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
|
|
378
|
+
}));
|
|
379
|
+
try {
|
|
380
|
+
await this.client.createTopics({ topics, waitForLeaders: true });
|
|
381
|
+
} catch (err) {
|
|
382
|
+
const e = err;
|
|
383
|
+
if (e?.type === "TOPIC_ALREADY_EXISTS") return;
|
|
384
|
+
if (/already exists/i.test(e?.message ?? "")) return;
|
|
385
|
+
throw err;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
async createPartitions(specs) {
|
|
389
|
+
if (specs.length === 0) return;
|
|
390
|
+
await this.client.createPartitions({
|
|
391
|
+
topicPartitions: specs.map((s) => ({
|
|
392
|
+
topic: s.topic,
|
|
393
|
+
count: s.totalCount
|
|
394
|
+
}))
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
};
|
|
305
398
|
async function importKafkaJs() {
|
|
306
399
|
try {
|
|
307
400
|
return await import("kafkajs");
|
|
@@ -434,6 +527,9 @@ function buildConfluentClientConfig(opts) {
|
|
|
434
527
|
if (opts.transactionTimeoutMs !== void 0) {
|
|
435
528
|
librdkafka["transaction.timeout.ms"] = opts.transactionTimeoutMs;
|
|
436
529
|
}
|
|
530
|
+
if (opts.compressionLevel !== void 0) {
|
|
531
|
+
librdkafka["compression.level"] = opts.compressionLevel;
|
|
532
|
+
}
|
|
437
533
|
const tlsRequested = opts.ssl === true || isTlsConfig(opts.ssl);
|
|
438
534
|
const saslRequested = !!opts.sasl;
|
|
439
535
|
if (saslRequested && tlsRequested) {
|
|
@@ -463,6 +559,9 @@ function buildConfluentClientConfig(opts) {
|
|
|
463
559
|
if (opts.sasl) {
|
|
464
560
|
kafkaJS["sasl"] = opts.sasl;
|
|
465
561
|
}
|
|
562
|
+
if (opts.rawProducerConfig) {
|
|
563
|
+
Object.assign(librdkafka, opts.rawProducerConfig);
|
|
564
|
+
}
|
|
466
565
|
return { kafkaJS, librdkafka };
|
|
467
566
|
}
|
|
468
567
|
function isTlsConfig(v) {
|
|
@@ -516,6 +615,17 @@ var ConfluentDriver = class {
|
|
|
516
615
|
await this.producer?.disconnect();
|
|
517
616
|
this.producer = null;
|
|
518
617
|
}
|
|
618
|
+
/**
|
|
619
|
+
* Construct a librdkafka-backed admin client wrapped in the eventferry
|
|
620
|
+
* `KafkaDriverAdmin` shape. The publisher's `connect()` is called before
|
|
621
|
+
* the admin reaches the user.
|
|
622
|
+
*/
|
|
623
|
+
async admin() {
|
|
624
|
+
const mod = await importConfluent();
|
|
625
|
+
const { kafkaJS, librdkafka } = buildConfluentClientConfig(this.opts);
|
|
626
|
+
const kafka = new mod.KafkaJS.Kafka({ kafkaJS, ...librdkafka });
|
|
627
|
+
return new ConfluentAdmin(kafka.admin());
|
|
628
|
+
}
|
|
519
629
|
async sendBatch(messages) {
|
|
520
630
|
if (!this.producer) throw new Error("ConfluentDriver not connected");
|
|
521
631
|
const topicMessages = groupByTopic2(messages);
|
|
@@ -584,6 +694,69 @@ function groupByTopic2(messages) {
|
|
|
584
694
|
messages: msgs
|
|
585
695
|
}));
|
|
586
696
|
}
|
|
697
|
+
var ConfluentAdmin = class {
|
|
698
|
+
constructor(client) {
|
|
699
|
+
this.client = client;
|
|
700
|
+
}
|
|
701
|
+
client;
|
|
702
|
+
async connect() {
|
|
703
|
+
await this.client.connect();
|
|
704
|
+
}
|
|
705
|
+
async close() {
|
|
706
|
+
await this.client.disconnect();
|
|
707
|
+
}
|
|
708
|
+
async listTopics() {
|
|
709
|
+
return await this.client.listTopics();
|
|
710
|
+
}
|
|
711
|
+
async describeTopics(topics) {
|
|
712
|
+
if (topics.length === 0) return [];
|
|
713
|
+
const all = new Set(await this.client.listTopics());
|
|
714
|
+
const existing = topics.filter((t) => all.has(t));
|
|
715
|
+
const missing = topics.filter((t) => !all.has(t));
|
|
716
|
+
const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
|
|
717
|
+
const byName = new Map(meta.topics.map((t) => [t.name, t]));
|
|
718
|
+
return topics.map((topic) => {
|
|
719
|
+
if (missing.includes(topic)) return { topic, partitions: [] };
|
|
720
|
+
const found = byName.get(topic);
|
|
721
|
+
if (!found) return { topic, partitions: [] };
|
|
722
|
+
return {
|
|
723
|
+
topic,
|
|
724
|
+
partitions: found.partitions.map((p) => ({
|
|
725
|
+
partitionId: p.partitionId,
|
|
726
|
+
leader: p.leader,
|
|
727
|
+
replicas: p.replicas,
|
|
728
|
+
isr: p.isr
|
|
729
|
+
}))
|
|
730
|
+
};
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
async createTopics(specs) {
|
|
734
|
+
if (specs.length === 0) return;
|
|
735
|
+
const topics = specs.map((s) => ({
|
|
736
|
+
topic: s.topic,
|
|
737
|
+
numPartitions: s.numPartitions,
|
|
738
|
+
replicationFactor: s.replicationFactor,
|
|
739
|
+
configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
|
|
740
|
+
}));
|
|
741
|
+
try {
|
|
742
|
+
await this.client.createTopics({ topics, waitForLeaders: true });
|
|
743
|
+
} catch (err) {
|
|
744
|
+
const e = err;
|
|
745
|
+
if (e?.code === 36 || e?.name === "TOPIC_ALREADY_EXISTS") return;
|
|
746
|
+
if (/already exists/i.test(e?.message ?? "")) return;
|
|
747
|
+
throw err;
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
async createPartitions(specs) {
|
|
751
|
+
if (specs.length === 0) return;
|
|
752
|
+
await this.client.createPartitions({
|
|
753
|
+
topicPartitions: specs.map((s) => ({
|
|
754
|
+
topic: s.topic,
|
|
755
|
+
count: s.totalCount
|
|
756
|
+
}))
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
};
|
|
587
760
|
async function importConfluent() {
|
|
588
761
|
try {
|
|
589
762
|
return await import("@confluentinc/kafka-javascript");
|
|
@@ -634,10 +807,12 @@ var KafkaPublisher = class {
|
|
|
634
807
|
logger;
|
|
635
808
|
hooks;
|
|
636
809
|
tracer;
|
|
810
|
+
validateTopicsOnConnect;
|
|
637
811
|
constructor(opts) {
|
|
638
812
|
this.logger = opts.logger;
|
|
639
813
|
this.hooks = opts.hooks ?? {};
|
|
640
814
|
this.tracer = opts.tracer ?? new NoopKafkaTracer();
|
|
815
|
+
this.validateTopicsOnConnect = opts.validateTopicsOnConnect ? Object.freeze([...opts.validateTopicsOnConnect]) : void 0;
|
|
641
816
|
const onTransactionAbort = this.hooks.onTransactionAbort ? (error) => {
|
|
642
817
|
void safeHook(
|
|
643
818
|
this.logger,
|
|
@@ -649,8 +824,90 @@ var KafkaPublisher = class {
|
|
|
649
824
|
}
|
|
650
825
|
async connect() {
|
|
651
826
|
await this.driver.connect();
|
|
827
|
+
if (this.validateTopicsOnConnect && this.validateTopicsOnConnect.length) {
|
|
828
|
+
await this.assertTopicsExist(this.validateTopicsOnConnect);
|
|
829
|
+
}
|
|
652
830
|
await safeHook(this.logger, "onConnect", () => this.hooks.onConnect?.());
|
|
653
831
|
}
|
|
832
|
+
/**
|
|
833
|
+
* Borrow a new admin client from the driver. The returned admin is
|
|
834
|
+
* connected and ready to use; the CALLER must `close()` it. Throws if the
|
|
835
|
+
* driver does not implement admin (custom driver lacking the capability).
|
|
836
|
+
*/
|
|
837
|
+
async admin() {
|
|
838
|
+
const driverAdmin = await this.openDriverAdmin();
|
|
839
|
+
return driverAdmin;
|
|
840
|
+
}
|
|
841
|
+
/**
|
|
842
|
+
* Idempotently provision topics. Each spec creates the topic if absent;
|
|
843
|
+
* existing topics are skipped without error. If `growPartitions: true`
|
|
844
|
+
* (default false), topics whose current partition count is below the
|
|
845
|
+
* requested `numPartitions` are grown via `createPartitions`.
|
|
846
|
+
*
|
|
847
|
+
* Replication factor and config entries on EXISTING topics are NOT
|
|
848
|
+
* reconciled — Kafka does not provide a safe in-place alter for those
|
|
849
|
+
* (changing replication requires reassignment; configs use alterConfigs).
|
|
850
|
+
* Reach for the raw admin if you need that.
|
|
851
|
+
*/
|
|
852
|
+
async ensureTopics(specs, opts = {}) {
|
|
853
|
+
if (specs.length === 0) return;
|
|
854
|
+
const admin = await this.openDriverAdmin();
|
|
855
|
+
try {
|
|
856
|
+
const topicNames = specs.map((s) => s.topic);
|
|
857
|
+
const existing = await admin.describeTopics(topicNames);
|
|
858
|
+
const existingByName = new Map(existing.map((t) => [t.topic, t]));
|
|
859
|
+
const toCreate = specs.filter(
|
|
860
|
+
(s) => (existingByName.get(s.topic)?.partitions.length ?? 0) === 0
|
|
861
|
+
);
|
|
862
|
+
if (toCreate.length) await admin.createTopics(toCreate);
|
|
863
|
+
if (opts.growPartitions) {
|
|
864
|
+
const grow = [];
|
|
865
|
+
for (const s of specs) {
|
|
866
|
+
if (s.numPartitions === void 0) continue;
|
|
867
|
+
const current = existingByName.get(s.topic);
|
|
868
|
+
const currentCount = current?.partitions.length ?? 0;
|
|
869
|
+
if (currentCount > 0 && currentCount < s.numPartitions) {
|
|
870
|
+
grow.push({ topic: s.topic, totalCount: s.numPartitions });
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
if (grow.length) await admin.createPartitions(grow);
|
|
874
|
+
}
|
|
875
|
+
} finally {
|
|
876
|
+
await admin.close();
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
/**
|
|
880
|
+
* Borrow a fresh admin from the driver and connect it. Throws when the
|
|
881
|
+
* driver does not implement admin (custom drivers without that capability).
|
|
882
|
+
*/
|
|
883
|
+
async openDriverAdmin() {
|
|
884
|
+
if (!this.driver.admin) {
|
|
885
|
+
throw new Error(
|
|
886
|
+
"KafkaPublisher: configured driver does not implement admin(). Use the built-in kafkajs or confluent driver, or extend your custom driver."
|
|
887
|
+
);
|
|
888
|
+
}
|
|
889
|
+
const admin = await this.driver.admin();
|
|
890
|
+
await admin.connect();
|
|
891
|
+
return admin;
|
|
892
|
+
}
|
|
893
|
+
/**
|
|
894
|
+
* Open an admin, list topics, throw if any required topic is missing.
|
|
895
|
+
* Always closes the admin (success or failure).
|
|
896
|
+
*/
|
|
897
|
+
async assertTopicsExist(required) {
|
|
898
|
+
const admin = await this.openDriverAdmin();
|
|
899
|
+
try {
|
|
900
|
+
const all = new Set(await admin.listTopics());
|
|
901
|
+
const missing = required.filter((t) => !all.has(t));
|
|
902
|
+
if (missing.length) {
|
|
903
|
+
throw new Error(
|
|
904
|
+
`KafkaPublisher: validateTopicsOnConnect failed \u2014 topics missing on cluster: ${missing.join(", ")}`
|
|
905
|
+
);
|
|
906
|
+
}
|
|
907
|
+
} finally {
|
|
908
|
+
await admin.close();
|
|
909
|
+
}
|
|
910
|
+
}
|
|
654
911
|
async disconnect() {
|
|
655
912
|
await this.driver.disconnect();
|
|
656
913
|
await safeHook(
|
|
@@ -662,9 +919,14 @@ var KafkaPublisher = class {
|
|
|
662
919
|
async publish(messages) {
|
|
663
920
|
if (messages.length === 0) return [];
|
|
664
921
|
const span = this.startBatchSpan(messages);
|
|
922
|
+
const outgoing = this.tracer.inject ? messages.map((m) => {
|
|
923
|
+
const headers = { ...m.headers };
|
|
924
|
+
this.tracer.inject(span, headers);
|
|
925
|
+
return { ...m, headers };
|
|
926
|
+
}) : messages;
|
|
665
927
|
let results;
|
|
666
928
|
try {
|
|
667
|
-
results = await this.driver.sendBatch(
|
|
929
|
+
results = await this.driver.sendBatch(outgoing);
|
|
668
930
|
} catch (err) {
|
|
669
931
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
670
932
|
span.setStatus({ code: "error", message: error.message });
|