@eventferry/kafka 3.3.1 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -1,5 +1,97 @@
1
1
  import { PublishableMessage, PublishResult, Logger, PublishErrorKind, Publisher } from '@eventferry/core';
2
2
 
3
+ /**
4
+ * Admin surface for `KafkaPublisher`. A typed wrapper over each driver's
5
+ * underlying admin client — implementations live in `kafkajs-driver.ts` and
6
+ * `confluent-driver.ts`. The publisher exposes this via `publisher.admin()`
7
+ * (returns a connected admin) and `publisher.ensureTopics()` (idempotent
8
+ * topic provisioning built on top).
9
+ *
10
+ * Scope is deliberately the most-used subset of the Kafka AdminClient
11
+ * protocol — listing, describing, creating topics and partitions. ACL
12
+ * management, quota inspection, and consumer-group operations are left to
13
+ * the underlying client (reach for kafkajs's `Admin` directly if needed).
14
+ */
15
+ /** Specification for creating one topic. */
16
+ interface TopicCreateSpec {
17
+ topic: string;
18
+ /** Default: cluster's `num.partitions` broker setting. */
19
+ numPartitions?: number;
20
+ /** Default: cluster's `default.replication.factor` broker setting. */
21
+ replicationFactor?: number;
22
+ /**
23
+ * Per-topic config entries (e.g. `{ "retention.ms": "604800000" }`).
24
+ * See Kafka broker docs for the full set.
25
+ */
26
+ configEntries?: Record<string, string>;
27
+ }
28
+ /** Topic + partition descriptor returned by describeTopics. */
29
+ interface TopicMetadata {
30
+ topic: string;
31
+ /** Empty when the topic doesn't exist (so callers can detect absence cheaply). */
32
+ partitions: PartitionMetadata[];
33
+ }
34
+ interface PartitionMetadata {
35
+ partitionId: number;
36
+ /** Broker id of the partition leader; -1 if no leader is known. */
37
+ leader: number;
38
+ /** Replica broker ids. */
39
+ replicas: number[];
40
+ /** In-sync replica broker ids. */
41
+ isr: number[];
42
+ }
43
+ /** Specification for growing a topic's partition count (never shrink). */
44
+ interface PartitionGrowSpec {
45
+ topic: string;
46
+ /** Total partition count after the change. MUST be ≥ the current count. */
47
+ totalCount: number;
48
+ }
49
+ /**
50
+ * Typed admin surface exposed by {@link KafkaPublisher.admin}.
51
+ *
52
+ * The returned object is already connected — call `.close()` (or let the
53
+ * publisher's `disconnect()` cascade close it) when done.
54
+ *
55
+ * All methods may throw native errors from the underlying client. The
56
+ * publisher does not classify admin errors via the `errorKind` machinery
57
+ * because admin failures are operator-facing and don't flow through the
58
+ * relay's retry path.
59
+ */
60
+ interface KafkaAdmin {
61
+ /** All topic names visible to this principal, including internal topics. */
62
+ listTopics(): Promise<string[]>;
63
+ /**
64
+ * Metadata for the given topics. Topics that don't exist on the cluster
65
+ * are returned with an empty `partitions` array — callers detect absence
66
+ * cheaply without a try/catch.
67
+ */
68
+ describeTopics(topics: string[]): Promise<TopicMetadata[]>;
69
+ /**
70
+ * Create topics. Idempotent at this layer: topics that already exist are
71
+ * silently skipped (the underlying client may throw `TopicExistsError`;
72
+ * we swallow it). Use `ensureTopics` on the publisher for partition-count
73
+ * + replication-factor coherence checks.
74
+ */
75
+ createTopics(specs: TopicCreateSpec[]): Promise<void>;
76
+ /**
77
+ * Grow each topic's partition count. Never shrinks (Kafka does not
78
+ * support shrinking partitions). Specs whose `totalCount` equals the
79
+ * current partition count are silently skipped.
80
+ */
81
+ createPartitions(specs: PartitionGrowSpec[]): Promise<void>;
82
+ /** Disconnect the admin client. */
83
+ close(): Promise<void>;
84
+ }
85
+ /**
86
+ * Optional driver-level hook returned by {@link KafkaDriver.admin}. Drivers
87
+ * implement this thin contract; the publisher composes the higher-level
88
+ * `KafkaAdmin` and `ensureTopics` on top.
89
+ */
90
+ interface KafkaDriverAdmin extends KafkaAdmin {
91
+ /** Called by the publisher before any method is invoked. */
92
+ connect(): Promise<void>;
93
+ }
94
+
3
95
  /**
4
96
  * Low-level driver contract. Each concrete driver (kafkajs, confluent)
5
97
  * adapts its native client to this minimal surface. The KafkaPublisher
@@ -19,6 +111,14 @@ interface KafkaDriver {
19
111
  * uses this to decide whether `sendBatch` is atomic.
20
112
  */
21
113
  readonly transactional: boolean;
114
+ /**
115
+ * Construct a NEW admin client. The returned admin is not yet connected —
116
+ * the publisher calls `.connect()` before handing it to the user.
117
+ *
118
+ * Optional: drivers without an admin surface may omit this; the publisher
119
+ * throws a clear error when `publisher.admin()` is called on such a driver.
120
+ */
121
+ admin?(): Promise<KafkaDriverAdmin>;
22
122
  }
23
123
  /**
24
124
  * TLS configuration for client connections. Pass a full {@link TlsConfig}
@@ -149,6 +249,14 @@ interface ProducerBehaviorConfig {
149
249
  acks?: number;
150
250
  /** Compression codec. Driver maps to its native enum. */
151
251
  compression?: "none" | "gzip" | "snappy" | "lz4" | "zstd";
252
+ /**
253
+ * (confluent only) Compression level for the chosen codec. Defaults vary
254
+ * per codec — librdkafka picks the broker-friendly default when unset.
255
+ * Common ranges: gzip 1–9, lz4 0–12, zstd 1–22 (higher = smaller + slower).
256
+ *
257
+ * No-op on the kafkajs driver (kafkajs does not expose codec levels).
258
+ */
259
+ compressionLevel?: number;
152
260
  /**
153
261
  * (confluent only) How long the producer waits to accumulate records before
154
262
  * flushing a partition batch. Default 0 (ship-immediately). Increase to
@@ -199,6 +307,36 @@ interface ProducerBehaviorConfig {
199
307
  * whether this callback throws.
200
308
  */
201
309
  onTransactionAbort?: (error: Error) => void;
310
+ /**
311
+ * (confluent only) Raw librdkafka producer-config keys merged on top of
312
+ * eventferry's translated config. Use for tuning surface area we don't
313
+ * expose typed (e.g. `queue.buffering.max.messages`, `socket.keepalive.enable`,
314
+ * `statistics.interval.ms`). Native keys win against the translated ones,
315
+ * so this can also be used to override defaults.
316
+ *
317
+ * Ignored by the kafkajs driver — log a one-time warning instead of
318
+ * silently dropping. Use `rawKafkaJsProducerConfig` for kafkajs-side tuning.
319
+ */
320
+ rawProducerConfig?: Record<string, unknown>;
321
+ /**
322
+ * (kafkajs only) Raw producer-config keys merged into kafkajs's
323
+ * `kafka.producer({...})` call. Native keys win against the translated
324
+ * ones. Use for kafkajs-internal knobs like `retry`, `metadataMaxAge`,
325
+ * `idempotent` overrides, etc.
326
+ *
327
+ * No-op on the confluent driver — use `rawProducerConfig` there.
328
+ */
329
+ rawKafkaJsProducerConfig?: Record<string, unknown>;
330
+ /**
331
+ * (kafkajs only) Custom partitioner factory passed straight to
332
+ * `kafka.producer({ createPartitioner })`. Overrides {@link partitioner}
333
+ * preset entirely. See kafkajs docs for the factory signature:
334
+ * `() => (args: { topic, partitionMetadata, message }) => number`.
335
+ *
336
+ * Ignored by the confluent driver — librdkafka's partitioner is a C
337
+ * extension point, not a JS callback.
338
+ */
339
+ customPartitioner?: () => (args: unknown) => number;
202
340
  }
203
341
  type DriverKind = "kafkajs" | "confluent";
204
342
 
@@ -213,6 +351,11 @@ interface KjsTransaction {
213
351
  commit(): Promise<void>;
214
352
  abort(): Promise<void>;
215
353
  }
354
+ interface KjsPartitionersNamespace {
355
+ DefaultPartitioner: () => unknown;
356
+ LegacyPartitioner: () => unknown;
357
+ JavaCompatiblePartitioner: () => unknown;
358
+ }
216
359
  interface KafkaJsDriverOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
217
360
  /**
218
361
  * Optional logger for the driver's own diagnostics (e.g. warnings about
@@ -235,7 +378,19 @@ declare class KafkaJsDriver implements KafkaDriver {
235
378
  * the send/transaction logic can be exercised without a real broker.
236
379
  */
237
380
  protected createProducer(): Promise<KjsProducer>;
381
+ /**
382
+ * Compute the options object passed to `kafka.producer({...})`. Exposed
383
+ * as a test seam so power-user escape hatches (customPartitioner,
384
+ * rawKafkaJsProducerConfig) can be asserted without a live broker.
385
+ */
386
+ protected buildProducerOptions(partitioners: KjsPartitionersNamespace | undefined): Promise<Record<string, unknown>>;
238
387
  disconnect(): Promise<void>;
388
+ /**
389
+ * Construct a kafkajs admin client wrapped in the eventferry-facing
390
+ * `KafkaDriverAdmin` shape. The publisher calls `.connect()` on the
391
+ * returned object before exposing it via `publisher.admin()`.
392
+ */
393
+ admin(): Promise<KafkaDriverAdmin>;
239
394
  sendBatch(messages: PublishableMessage[]): Promise<PublishResult[]>;
240
395
  }
241
396
  /** Internal — used by tests. Resets the dedup so warnings can be observed in isolation. */
@@ -288,6 +443,12 @@ declare class ConfluentDriver implements KafkaDriver {
288
443
  */
289
444
  protected createProducer(): Promise<CkProducer>;
290
445
  disconnect(): Promise<void>;
446
+ /**
447
+ * Construct a librdkafka-backed admin client wrapped in the eventferry
448
+ * `KafkaDriverAdmin` shape. The publisher's `connect()` is called before
449
+ * the admin reaches the user.
450
+ */
451
+ admin(): Promise<KafkaDriverAdmin>;
291
452
  sendBatch(messages: PublishableMessage[]): Promise<PublishResult[]>;
292
453
  }
293
454
 
@@ -423,6 +584,21 @@ interface KafkaTracer {
423
584
  * server.address/port).
424
585
  */
425
586
  startPublishSpan(name: string, attributes: Record<string, SpanAttributeValue>): SpanLike;
587
+ /**
588
+ * OPTIONAL: inject the active trace context (W3C `traceparent` +
589
+ * `tracestate`) into a per-message header map. Called by the publisher
590
+ * AFTER the batch span is created and BEFORE the records hit the wire.
591
+ *
592
+ * Implementations typically wrap OpenTelemetry's `propagation.inject(...)`
593
+ * or your tracing SDK's equivalent. Mutate the `headers` object in
594
+ * place — the publisher allocates a fresh copy per message so this is
595
+ * safe and matches the propagation API of every major SDK.
596
+ *
597
+ * Tracers without distributed-context propagation (or that only care
598
+ * about local spans) may leave this off — consumers can still derive
599
+ * trace headers themselves by other means.
600
+ */
601
+ inject?(span: SpanLike, headers: Record<string, string>): void;
426
602
  }
427
603
  /**
428
604
  * No-op tracer. Used when the user does not configure one. Cheap allocation
@@ -457,6 +633,16 @@ interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorC
457
633
  * Use a thin adapter over your tracing SDK (see {@link KafkaTracer}).
458
634
  */
459
635
  tracer?: KafkaTracer;
636
+ /**
637
+ * If set, `connect()` checks that every topic in this list exists on the
638
+ * cluster and throws a descriptive error if any are missing. Use this to
639
+ * fail-fast at startup instead of letting the first send-time error
640
+ * surprise you.
641
+ *
642
+ * Validation runs AFTER the producer connects but BEFORE `onConnect` hooks
643
+ * fire. Driver must implement `admin()` (the built-ins do).
644
+ */
645
+ validateTopicsOnConnect?: string[];
460
646
  }
461
647
  /**
462
648
  * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and adds
@@ -469,8 +655,39 @@ declare class KafkaPublisher implements Publisher {
469
655
  private readonly logger;
470
656
  private readonly hooks;
471
657
  private readonly tracer;
658
+ private readonly validateTopicsOnConnect;
472
659
  constructor(opts: KafkaPublisherOptions);
473
660
  connect(): Promise<void>;
661
+ /**
662
+ * Borrow a new admin client from the driver. The returned admin is
663
+ * connected and ready to use; the CALLER must `close()` it. Throws if the
664
+ * driver does not implement admin (custom driver lacking the capability).
665
+ */
666
+ admin(): Promise<KafkaAdmin>;
667
+ /**
668
+ * Idempotently provision topics. Each spec creates the topic if absent;
669
+ * existing topics are skipped without error. If `growPartitions: true`
670
+ * (default false), topics whose current partition count is below the
671
+ * requested `numPartitions` are grown via `createPartitions`.
672
+ *
673
+ * Replication factor and config entries on EXISTING topics are NOT
674
+ * reconciled — Kafka does not provide a safe in-place alter for those
675
+ * (changing replication requires reassignment; configs use alterConfigs).
676
+ * Reach for the raw admin if you need that.
677
+ */
678
+ ensureTopics(specs: TopicCreateSpec[], opts?: {
679
+ growPartitions?: boolean;
680
+ }): Promise<void>;
681
+ /**
682
+ * Borrow a fresh admin from the driver and connect it. Throws when the
683
+ * driver does not implement admin (custom drivers without that capability).
684
+ */
685
+ private openDriverAdmin;
686
+ /**
687
+ * Open an admin, list topics, throw if any required topic is missing.
688
+ * Always closes the admin (success or failure).
689
+ */
690
+ private assertTopicsExist;
474
691
  disconnect(): Promise<void>;
475
692
  publish(messages: PublishableMessage[]): Promise<PublishResult[]>;
476
693
  /**
@@ -491,4 +708,4 @@ declare class KafkaPublisher implements Publisher {
491
708
  private startBatchSpan;
492
709
  }
493
710
 
494
- export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };
711
+ export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaAdmin, type KafkaConnectionConfig, type KafkaDriver, type KafkaDriverAdmin, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type PartitionGrowSpec, type PartitionMetadata, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, type TopicCreateSpec, type TopicMetadata, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };
package/dist/index.js CHANGED
@@ -104,7 +104,10 @@ var UNSUPPORTED_BY_KAFKAJS = [
104
104
  "lingerMs",
105
105
  "batchSize",
106
106
  "deliveryTimeoutMs",
107
- "maxRequestSize"
107
+ "maxRequestSize",
108
+ // Confluent-only escape hatches; ignored on kafkajs.
109
+ "compressionLevel",
110
+ "rawProducerConfig"
108
111
  ];
109
112
  var KafkaJsDriver = class {
110
113
  transactional;
@@ -143,13 +146,21 @@ var KafkaJsDriver = class {
143
146
  // the provider's returned token (other fields are ignored).
144
147
  sasl: this.opts.sasl
145
148
  });
146
- const createPartitioner = resolveCreatePartitioner(
147
- mod.Partitioners,
149
+ return kafka.producer(await this.buildProducerOptions(mod.Partitioners));
150
+ }
151
+ /**
152
+ * Compute the options object passed to `kafka.producer({...})`. Exposed
153
+ * as a test seam so power-user escape hatches (customPartitioner,
154
+ * rawKafkaJsProducerConfig) can be asserted without a live broker.
155
+ */
156
+ async buildProducerOptions(partitioners) {
157
+ const createPartitioner = this.opts.customPartitioner ?? resolveCreatePartitioner(
158
+ partitioners,
148
159
  this.opts.partitioner,
149
160
  this.transactional
150
161
  );
151
162
  const resolvedTxId = this.transactional ? await resolveTransactionalId(this.opts.transactionalId) : void 0;
152
- return kafka.producer({
163
+ return {
153
164
  idempotent: this.opts.idempotent ?? true,
154
165
  // Idempotent / transactional producers cap maxInFlight at 5. When the
155
166
  // user picks transactional we force 1 to keep strict ordering across
@@ -163,13 +174,32 @@ var KafkaJsDriver = class {
163
174
  transactionTimeout: this.opts.transactionTimeoutMs,
164
175
  // Setting any partitioner choice silences kafkajs's
165
176
  // KafkaJSPartitionerNotSpecified warning.
166
- createPartitioner
167
- });
177
+ createPartitioner,
178
+ // Power-user escape hatch — merged LAST so raw keys win against the
179
+ // translated ones. That's the contract: anything you put here is
180
+ // final, even if it overrides idempotent/transactionalId/etc.
181
+ ...this.opts.rawKafkaJsProducerConfig ?? {}
182
+ };
168
183
  }
169
184
  async disconnect() {
170
185
  await this.producer?.disconnect();
171
186
  this.producer = null;
172
187
  }
188
+ /**
189
+ * Construct a kafkajs admin client wrapped in the eventferry-facing
190
+ * `KafkaDriverAdmin` shape. The publisher calls `.connect()` on the
191
+ * returned object before exposing it via `publisher.admin()`.
192
+ */
193
+ async admin() {
194
+ const mod = await importKafkaJs();
195
+ const kafka = new mod.Kafka({
196
+ clientId: this.opts.clientId ?? "eventferry-admin",
197
+ brokers: this.opts.brokers,
198
+ ssl: this.opts.ssl,
199
+ sasl: this.opts.sasl
200
+ });
201
+ return new KafkaJsAdmin(kafka.admin());
202
+ }
173
203
  async sendBatch(messages) {
174
204
  if (!this.producer) throw new Error("KafkaJsDriver not connected");
175
205
  const topicMessages = groupByTopic(messages, this.opts.compression);
@@ -258,6 +288,69 @@ function warnUnsupportedKafkajsOptions(opts) {
258
288
  function _resetKafkajsWarnDedup() {
259
289
  warnedKafkajsKeys.clear();
260
290
  }
291
+ var KafkaJsAdmin = class {
292
+ constructor(client) {
293
+ this.client = client;
294
+ }
295
+ client;
296
+ async connect() {
297
+ await this.client.connect();
298
+ }
299
+ async close() {
300
+ await this.client.disconnect();
301
+ }
302
+ async listTopics() {
303
+ return await this.client.listTopics();
304
+ }
305
+ async describeTopics(topics) {
306
+ if (topics.length === 0) return [];
307
+ const all = new Set(await this.client.listTopics());
308
+ const existing = topics.filter((t) => all.has(t));
309
+ const missing = topics.filter((t) => !all.has(t));
310
+ const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
311
+ const byName = new Map(meta.topics.map((t) => [t.name, t]));
312
+ return topics.map((topic) => {
313
+ if (missing.includes(topic)) return { topic, partitions: [] };
314
+ const found = byName.get(topic);
315
+ if (!found) return { topic, partitions: [] };
316
+ return {
317
+ topic,
318
+ partitions: found.partitions.map((p) => ({
319
+ partitionId: p.partitionId,
320
+ leader: p.leader,
321
+ replicas: p.replicas,
322
+ isr: p.isr
323
+ }))
324
+ };
325
+ });
326
+ }
327
+ async createTopics(specs) {
328
+ if (specs.length === 0) return;
329
+ const topics = specs.map((s) => ({
330
+ topic: s.topic,
331
+ numPartitions: s.numPartitions,
332
+ replicationFactor: s.replicationFactor,
333
+ configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
334
+ }));
335
+ try {
336
+ await this.client.createTopics({ topics, waitForLeaders: true });
337
+ } catch (err) {
338
+ const e = err;
339
+ if (e?.type === "TOPIC_ALREADY_EXISTS") return;
340
+ if (/already exists/i.test(e?.message ?? "")) return;
341
+ throw err;
342
+ }
343
+ }
344
+ async createPartitions(specs) {
345
+ if (specs.length === 0) return;
346
+ await this.client.createPartitions({
347
+ topicPartitions: specs.map((s) => ({
348
+ topic: s.topic,
349
+ count: s.totalCount
350
+ }))
351
+ });
352
+ }
353
+ };
261
354
  async function importKafkaJs() {
262
355
  try {
263
356
  return await import("kafkajs");
@@ -390,6 +483,9 @@ function buildConfluentClientConfig(opts) {
390
483
  if (opts.transactionTimeoutMs !== void 0) {
391
484
  librdkafka["transaction.timeout.ms"] = opts.transactionTimeoutMs;
392
485
  }
486
+ if (opts.compressionLevel !== void 0) {
487
+ librdkafka["compression.level"] = opts.compressionLevel;
488
+ }
393
489
  const tlsRequested = opts.ssl === true || isTlsConfig(opts.ssl);
394
490
  const saslRequested = !!opts.sasl;
395
491
  if (saslRequested && tlsRequested) {
@@ -419,6 +515,9 @@ function buildConfluentClientConfig(opts) {
419
515
  if (opts.sasl) {
420
516
  kafkaJS["sasl"] = opts.sasl;
421
517
  }
518
+ if (opts.rawProducerConfig) {
519
+ Object.assign(librdkafka, opts.rawProducerConfig);
520
+ }
422
521
  return { kafkaJS, librdkafka };
423
522
  }
424
523
  function isTlsConfig(v) {
@@ -472,6 +571,17 @@ var ConfluentDriver = class {
472
571
  await this.producer?.disconnect();
473
572
  this.producer = null;
474
573
  }
574
+ /**
575
+ * Construct a librdkafka-backed admin client wrapped in the eventferry
576
+ * `KafkaDriverAdmin` shape. The publisher's `connect()` is called before
577
+ * the admin reaches the user.
578
+ */
579
+ async admin() {
580
+ const mod = await importConfluent();
581
+ const { kafkaJS, librdkafka } = buildConfluentClientConfig(this.opts);
582
+ const kafka = new mod.KafkaJS.Kafka({ kafkaJS, ...librdkafka });
583
+ return new ConfluentAdmin(kafka.admin());
584
+ }
475
585
  async sendBatch(messages) {
476
586
  if (!this.producer) throw new Error("ConfluentDriver not connected");
477
587
  const topicMessages = groupByTopic2(messages);
@@ -540,6 +650,69 @@ function groupByTopic2(messages) {
540
650
  messages: msgs
541
651
  }));
542
652
  }
653
+ var ConfluentAdmin = class {
654
+ constructor(client) {
655
+ this.client = client;
656
+ }
657
+ client;
658
+ async connect() {
659
+ await this.client.connect();
660
+ }
661
+ async close() {
662
+ await this.client.disconnect();
663
+ }
664
+ async listTopics() {
665
+ return await this.client.listTopics();
666
+ }
667
+ async describeTopics(topics) {
668
+ if (topics.length === 0) return [];
669
+ const all = new Set(await this.client.listTopics());
670
+ const existing = topics.filter((t) => all.has(t));
671
+ const missing = topics.filter((t) => !all.has(t));
672
+ const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
673
+ const byName = new Map(meta.topics.map((t) => [t.name, t]));
674
+ return topics.map((topic) => {
675
+ if (missing.includes(topic)) return { topic, partitions: [] };
676
+ const found = byName.get(topic);
677
+ if (!found) return { topic, partitions: [] };
678
+ return {
679
+ topic,
680
+ partitions: found.partitions.map((p) => ({
681
+ partitionId: p.partitionId,
682
+ leader: p.leader,
683
+ replicas: p.replicas,
684
+ isr: p.isr
685
+ }))
686
+ };
687
+ });
688
+ }
689
+ async createTopics(specs) {
690
+ if (specs.length === 0) return;
691
+ const topics = specs.map((s) => ({
692
+ topic: s.topic,
693
+ numPartitions: s.numPartitions,
694
+ replicationFactor: s.replicationFactor,
695
+ configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
696
+ }));
697
+ try {
698
+ await this.client.createTopics({ topics, waitForLeaders: true });
699
+ } catch (err) {
700
+ const e = err;
701
+ if (e?.code === 36 || e?.name === "TOPIC_ALREADY_EXISTS") return;
702
+ if (/already exists/i.test(e?.message ?? "")) return;
703
+ throw err;
704
+ }
705
+ }
706
+ async createPartitions(specs) {
707
+ if (specs.length === 0) return;
708
+ await this.client.createPartitions({
709
+ topicPartitions: specs.map((s) => ({
710
+ topic: s.topic,
711
+ count: s.totalCount
712
+ }))
713
+ });
714
+ }
715
+ };
543
716
  async function importConfluent() {
544
717
  try {
545
718
  return await import("@confluentinc/kafka-javascript");
@@ -590,10 +763,12 @@ var KafkaPublisher = class {
590
763
  logger;
591
764
  hooks;
592
765
  tracer;
766
+ validateTopicsOnConnect;
593
767
  constructor(opts) {
594
768
  this.logger = opts.logger;
595
769
  this.hooks = opts.hooks ?? {};
596
770
  this.tracer = opts.tracer ?? new NoopKafkaTracer();
771
+ this.validateTopicsOnConnect = opts.validateTopicsOnConnect ? Object.freeze([...opts.validateTopicsOnConnect]) : void 0;
597
772
  const onTransactionAbort = this.hooks.onTransactionAbort ? (error) => {
598
773
  void safeHook(
599
774
  this.logger,
@@ -605,8 +780,90 @@ var KafkaPublisher = class {
605
780
  }
606
781
  async connect() {
607
782
  await this.driver.connect();
783
+ if (this.validateTopicsOnConnect && this.validateTopicsOnConnect.length) {
784
+ await this.assertTopicsExist(this.validateTopicsOnConnect);
785
+ }
608
786
  await safeHook(this.logger, "onConnect", () => this.hooks.onConnect?.());
609
787
  }
788
+ /**
789
+ * Borrow a new admin client from the driver. The returned admin is
790
+ * connected and ready to use; the CALLER must `close()` it. Throws if the
791
+ * driver does not implement admin (custom driver lacking the capability).
792
+ */
793
+ async admin() {
794
+ const driverAdmin = await this.openDriverAdmin();
795
+ return driverAdmin;
796
+ }
797
+ /**
798
+ * Idempotently provision topics. Each spec creates the topic if absent;
799
+ * existing topics are skipped without error. If `growPartitions: true`
800
+ * (default false), topics whose current partition count is below the
801
+ * requested `numPartitions` are grown via `createPartitions`.
802
+ *
803
+ * Replication factor and config entries on EXISTING topics are NOT
804
+ * reconciled — Kafka does not provide a safe in-place alter for those
805
+ * (changing replication requires reassignment; configs use alterConfigs).
806
+ * Reach for the raw admin if you need that.
807
+ */
808
+ async ensureTopics(specs, opts = {}) {
809
+ if (specs.length === 0) return;
810
+ const admin = await this.openDriverAdmin();
811
+ try {
812
+ const topicNames = specs.map((s) => s.topic);
813
+ const existing = await admin.describeTopics(topicNames);
814
+ const existingByName = new Map(existing.map((t) => [t.topic, t]));
815
+ const toCreate = specs.filter(
816
+ (s) => (existingByName.get(s.topic)?.partitions.length ?? 0) === 0
817
+ );
818
+ if (toCreate.length) await admin.createTopics(toCreate);
819
+ if (opts.growPartitions) {
820
+ const grow = [];
821
+ for (const s of specs) {
822
+ if (s.numPartitions === void 0) continue;
823
+ const current = existingByName.get(s.topic);
824
+ const currentCount = current?.partitions.length ?? 0;
825
+ if (currentCount > 0 && currentCount < s.numPartitions) {
826
+ grow.push({ topic: s.topic, totalCount: s.numPartitions });
827
+ }
828
+ }
829
+ if (grow.length) await admin.createPartitions(grow);
830
+ }
831
+ } finally {
832
+ await admin.close();
833
+ }
834
+ }
835
+ /**
836
+ * Borrow a fresh admin from the driver and connect it. Throws when the
837
+ * driver does not implement admin (custom drivers without that capability).
838
+ */
839
+ async openDriverAdmin() {
840
+ if (!this.driver.admin) {
841
+ throw new Error(
842
+ "KafkaPublisher: configured driver does not implement admin(). Use the built-in kafkajs or confluent driver, or extend your custom driver."
843
+ );
844
+ }
845
+ const admin = await this.driver.admin();
846
+ await admin.connect();
847
+ return admin;
848
+ }
849
+ /**
850
+ * Open an admin, list topics, throw if any required topic is missing.
851
+ * Always closes the admin (success or failure).
852
+ */
853
+ async assertTopicsExist(required) {
854
+ const admin = await this.openDriverAdmin();
855
+ try {
856
+ const all = new Set(await admin.listTopics());
857
+ const missing = required.filter((t) => !all.has(t));
858
+ if (missing.length) {
859
+ throw new Error(
860
+ `KafkaPublisher: validateTopicsOnConnect failed \u2014 topics missing on cluster: ${missing.join(", ")}`
861
+ );
862
+ }
863
+ } finally {
864
+ await admin.close();
865
+ }
866
+ }
610
867
  async disconnect() {
611
868
  await this.driver.disconnect();
612
869
  await safeHook(
@@ -618,9 +875,14 @@ var KafkaPublisher = class {
618
875
  async publish(messages) {
619
876
  if (messages.length === 0) return [];
620
877
  const span = this.startBatchSpan(messages);
878
+ const outgoing = this.tracer.inject ? messages.map((m) => {
879
+ const headers = { ...m.headers };
880
+ this.tracer.inject(span, headers);
881
+ return { ...m, headers };
882
+ }) : messages;
621
883
  let results;
622
884
  try {
623
- results = await this.driver.sendBatch(messages);
885
+ results = await this.driver.sendBatch(outgoing);
624
886
  } catch (err) {
625
887
  const error = err instanceof Error ? err : new Error(String(err));
626
888
  span.setStatus({ code: "error", message: error.message });