@eventferry/kafka 3.3.0 → 3.4.0

package/dist/index.cjs

@@ -148,7 +148,10 @@ var UNSUPPORTED_BY_KAFKAJS = [
   "lingerMs",
   "batchSize",
   "deliveryTimeoutMs",
-  "maxRequestSize"
+  "maxRequestSize",
+  // Confluent-only escape hatches; ignored on kafkajs.
+  "compressionLevel",
+  "rawProducerConfig"
 ];
 var KafkaJsDriver = class {
   transactional;
@@ -187,13 +190,21 @@ var KafkaJsDriver = class {
       // the provider's returned token (other fields are ignored).
       sasl: this.opts.sasl
     });
-    const createPartitioner = resolveCreatePartitioner(
-      mod.Partitioners,
+    return kafka.producer(await this.buildProducerOptions(mod.Partitioners));
+  }
+  /**
+   * Compute the options object passed to `kafka.producer({...})`. Exposed
+   * as a test seam so power-user escape hatches (customPartitioner,
+   * rawKafkaJsProducerConfig) can be asserted without a live broker.
+   */
+  async buildProducerOptions(partitioners) {
+    const createPartitioner = this.opts.customPartitioner ?? resolveCreatePartitioner(
+      partitioners,
       this.opts.partitioner,
       this.transactional
     );
     const resolvedTxId = this.transactional ? await resolveTransactionalId(this.opts.transactionalId) : void 0;
-    return kafka.producer({
+    return {
       idempotent: this.opts.idempotent ?? true,
       // Idempotent / transactional producers cap maxInFlight at 5. When the
       // user picks transactional we force 1 to keep strict ordering across
@@ -207,13 +218,32 @@ var KafkaJsDriver = class {
       transactionTimeout: this.opts.transactionTimeoutMs,
       // Setting any partitioner choice silences kafkajs's
       // KafkaJSPartitionerNotSpecified warning.
-      createPartitioner
-    });
+      createPartitioner,
+      // Power-user escape hatch — merged LAST so raw keys win against the
+      // translated ones. That's the contract: anything you put here is
+      // final, even if it overrides idempotent/transactionalId/etc.
+      ...this.opts.rawKafkaJsProducerConfig ?? {}
+    };
   }
   async disconnect() {
     await this.producer?.disconnect();
     this.producer = null;
   }
+  /**
+   * Construct a kafkajs admin client wrapped in the eventferry-facing
+   * `KafkaDriverAdmin` shape. The publisher calls `.connect()` on the
+   * returned object before exposing it via `publisher.admin()`.
+   */
+  async admin() {
+    const mod = await importKafkaJs();
+    const kafka = new mod.Kafka({
+      clientId: this.opts.clientId ?? "eventferry-admin",
+      brokers: this.opts.brokers,
+      ssl: this.opts.ssl,
+      sasl: this.opts.sasl
+    });
+    return new KafkaJsAdmin(kafka.admin());
+  }
   async sendBatch(messages) {
     if (!this.producer) throw new Error("KafkaJsDriver not connected");
     const topicMessages = groupByTopic(messages, this.opts.compression);
@@ -302,6 +332,69 @@ function warnUnsupportedKafkajsOptions(opts) {
 function _resetKafkajsWarnDedup() {
   warnedKafkajsKeys.clear();
 }
+var KafkaJsAdmin = class {
+  constructor(client) {
+    this.client = client;
+  }
+  client;
+  async connect() {
+    await this.client.connect();
+  }
+  async close() {
+    await this.client.disconnect();
+  }
+  async listTopics() {
+    return await this.client.listTopics();
+  }
+  async describeTopics(topics) {
+    if (topics.length === 0) return [];
+    const all = new Set(await this.client.listTopics());
+    const existing = topics.filter((t) => all.has(t));
+    const missing = topics.filter((t) => !all.has(t));
+    const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
+    const byName = new Map(meta.topics.map((t) => [t.name, t]));
+    return topics.map((topic) => {
+      if (missing.includes(topic)) return { topic, partitions: [] };
+      const found = byName.get(topic);
+      if (!found) return { topic, partitions: [] };
+      return {
+        topic,
+        partitions: found.partitions.map((p) => ({
+          partitionId: p.partitionId,
+          leader: p.leader,
+          replicas: p.replicas,
+          isr: p.isr
+        }))
+      };
+    });
+  }
+  async createTopics(specs) {
+    if (specs.length === 0) return;
+    const topics = specs.map((s) => ({
+      topic: s.topic,
+      numPartitions: s.numPartitions,
+      replicationFactor: s.replicationFactor,
+      configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
+    }));
+    try {
+      await this.client.createTopics({ topics, waitForLeaders: true });
+    } catch (err) {
+      const e = err;
+      if (e?.type === "TOPIC_ALREADY_EXISTS") return;
+      if (/already exists/i.test(e?.message ?? "")) return;
+      throw err;
+    }
+  }
+  async createPartitions(specs) {
+    if (specs.length === 0) return;
+    await this.client.createPartitions({
+      topicPartitions: specs.map((s) => ({
+        topic: s.topic,
+        count: s.totalCount
+      }))
+    });
+  }
+};
 async function importKafkaJs() {
   try {
     return await import("kafkajs");
@@ -434,6 +527,9 @@ function buildConfluentClientConfig(opts) {
   if (opts.transactionTimeoutMs !== void 0) {
     librdkafka["transaction.timeout.ms"] = opts.transactionTimeoutMs;
   }
+  if (opts.compressionLevel !== void 0) {
+    librdkafka["compression.level"] = opts.compressionLevel;
+  }
   const tlsRequested = opts.ssl === true || isTlsConfig(opts.ssl);
   const saslRequested = !!opts.sasl;
   if (saslRequested && tlsRequested) {
@@ -463,6 +559,9 @@ function buildConfluentClientConfig(opts) {
   if (opts.sasl) {
     kafkaJS["sasl"] = opts.sasl;
   }
+  if (opts.rawProducerConfig) {
+    Object.assign(librdkafka, opts.rawProducerConfig);
+  }
   return { kafkaJS, librdkafka };
 }
 function isTlsConfig(v) {
@@ -516,6 +615,17 @@ var ConfluentDriver = class {
     await this.producer?.disconnect();
     this.producer = null;
   }
+  /**
+   * Construct a librdkafka-backed admin client wrapped in the eventferry
+   * `KafkaDriverAdmin` shape. The publisher's `connect()` is called before
+   * the admin reaches the user.
+   */
+  async admin() {
+    const mod = await importConfluent();
+    const { kafkaJS, librdkafka } = buildConfluentClientConfig(this.opts);
+    const kafka = new mod.KafkaJS.Kafka({ kafkaJS, ...librdkafka });
+    return new ConfluentAdmin(kafka.admin());
+  }
   async sendBatch(messages) {
     if (!this.producer) throw new Error("ConfluentDriver not connected");
     const topicMessages = groupByTopic2(messages);
@@ -584,6 +694,69 @@ function groupByTopic2(messages) {
     messages: msgs
   }));
 }
+var ConfluentAdmin = class {
+  constructor(client) {
+    this.client = client;
+  }
+  client;
+  async connect() {
+    await this.client.connect();
+  }
+  async close() {
+    await this.client.disconnect();
+  }
+  async listTopics() {
+    return await this.client.listTopics();
+  }
+  async describeTopics(topics) {
+    if (topics.length === 0) return [];
+    const all = new Set(await this.client.listTopics());
+    const existing = topics.filter((t) => all.has(t));
+    const missing = topics.filter((t) => !all.has(t));
+    const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
+    const byName = new Map(meta.topics.map((t) => [t.name, t]));
+    return topics.map((topic) => {
+      if (missing.includes(topic)) return { topic, partitions: [] };
+      const found = byName.get(topic);
+      if (!found) return { topic, partitions: [] };
+      return {
+        topic,
+        partitions: found.partitions.map((p) => ({
+          partitionId: p.partitionId,
+          leader: p.leader,
+          replicas: p.replicas,
+          isr: p.isr
+        }))
+      };
+    });
+  }
+  async createTopics(specs) {
+    if (specs.length === 0) return;
+    const topics = specs.map((s) => ({
+      topic: s.topic,
+      numPartitions: s.numPartitions,
+      replicationFactor: s.replicationFactor,
+      configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
+    }));
+    try {
+      await this.client.createTopics({ topics, waitForLeaders: true });
+    } catch (err) {
+      const e = err;
+      if (e?.code === 36 || e?.name === "TOPIC_ALREADY_EXISTS") return;
+      if (/already exists/i.test(e?.message ?? "")) return;
+      throw err;
+    }
+  }
+  async createPartitions(specs) {
+    if (specs.length === 0) return;
+    await this.client.createPartitions({
+      topicPartitions: specs.map((s) => ({
+        topic: s.topic,
+        count: s.totalCount
+      }))
+    });
+  }
+};
 async function importConfluent() {
   try {
     return await import("@confluentinc/kafka-javascript");
@@ -634,10 +807,12 @@ var KafkaPublisher = class {
   logger;
   hooks;
   tracer;
+  validateTopicsOnConnect;
   constructor(opts) {
     this.logger = opts.logger;
     this.hooks = opts.hooks ?? {};
     this.tracer = opts.tracer ?? new NoopKafkaTracer();
+    this.validateTopicsOnConnect = opts.validateTopicsOnConnect ? Object.freeze([...opts.validateTopicsOnConnect]) : void 0;
     const onTransactionAbort = this.hooks.onTransactionAbort ? (error) => {
       void safeHook(
         this.logger,
@@ -649,8 +824,90 @@ var KafkaPublisher = class {
   }
   async connect() {
     await this.driver.connect();
+    if (this.validateTopicsOnConnect && this.validateTopicsOnConnect.length) {
+      await this.assertTopicsExist(this.validateTopicsOnConnect);
+    }
     await safeHook(this.logger, "onConnect", () => this.hooks.onConnect?.());
   }
+  /**
+   * Borrow a new admin client from the driver. The returned admin is
+   * connected and ready to use; the CALLER must `close()` it. Throws if the
+   * driver does not implement admin (custom driver lacking the capability).
+   */
+  async admin() {
+    const driverAdmin = await this.openDriverAdmin();
+    return driverAdmin;
+  }
+  /**
+   * Idempotently provision topics. Each spec creates the topic if absent;
+   * existing topics are skipped without error. If `growPartitions: true`
+   * (default false), topics whose current partition count is below the
+   * requested `numPartitions` are grown via `createPartitions`.
+   *
+   * Replication factor and config entries on EXISTING topics are NOT
+   * reconciled — Kafka does not provide a safe in-place alter for those
+   * (changing replication requires reassignment; configs use alterConfigs).
+   * Reach for the raw admin if you need that.
+   */
+  async ensureTopics(specs, opts = {}) {
+    if (specs.length === 0) return;
+    const admin = await this.openDriverAdmin();
+    try {
+      const topicNames = specs.map((s) => s.topic);
+      const existing = await admin.describeTopics(topicNames);
+      const existingByName = new Map(existing.map((t) => [t.topic, t]));
+      const toCreate = specs.filter(
+        (s) => (existingByName.get(s.topic)?.partitions.length ?? 0) === 0
+      );
+      if (toCreate.length) await admin.createTopics(toCreate);
+      if (opts.growPartitions) {
+        const grow = [];
+        for (const s of specs) {
+          if (s.numPartitions === void 0) continue;
+          const current = existingByName.get(s.topic);
+          const currentCount = current?.partitions.length ?? 0;
+          if (currentCount > 0 && currentCount < s.numPartitions) {
+            grow.push({ topic: s.topic, totalCount: s.numPartitions });
+          }
+        }
+        if (grow.length) await admin.createPartitions(grow);
+      }
+    } finally {
+      await admin.close();
+    }
+  }
+  /**
+   * Borrow a fresh admin from the driver and connect it. Throws when the
+   * driver does not implement admin (custom drivers without that capability).
+   */
+  async openDriverAdmin() {
+    if (!this.driver.admin) {
+      throw new Error(
+        "KafkaPublisher: configured driver does not implement admin(). Use the built-in kafkajs or confluent driver, or extend your custom driver."
+      );
+    }
+    const admin = await this.driver.admin();
+    await admin.connect();
+    return admin;
+  }
+  /**
+   * Open an admin, list topics, throw if any required topic is missing.
+   * Always closes the admin (success or failure).
+   */
+  async assertTopicsExist(required) {
+    const admin = await this.openDriverAdmin();
+    try {
+      const all = new Set(await admin.listTopics());
+      const missing = required.filter((t) => !all.has(t));
+      if (missing.length) {
+        throw new Error(
+          `KafkaPublisher: validateTopicsOnConnect failed \u2014 topics missing on cluster: ${missing.join(", ")}`
+        );
+      }
+    } finally {
+      await admin.close();
+    }
+  }
   async disconnect() {
     await this.driver.disconnect();
     await safeHook(
@@ -662,9 +919,14 @@ var KafkaPublisher = class {
   async publish(messages) {
     if (messages.length === 0) return [];
     const span = this.startBatchSpan(messages);
+    const outgoing = this.tracer.inject ? messages.map((m) => {
+      const headers = { ...m.headers };
+      this.tracer.inject(span, headers);
+      return { ...m, headers };
+    }) : messages;
     let results;
     try {
-      results = await this.driver.sendBatch(messages);
+      results = await this.driver.sendBatch(outgoing);
     } catch (err) {
       const error = err instanceof Error ? err : new Error(String(err));
       span.setStatus({ code: "error", message: error.message });