@eventferry/kafka 3.3.1 → 3.5.0

package/dist/index.js

@@ -7,6 +7,7 @@ function classifyKafkajsError(err) {
   if (e.name === "KafkaJSNonRetriableError") return "fatal";
   const type = typeof e.type === "string" ? e.type : void 0;
   if (type) {
+    if (FENCED_TYPES.has(type)) return "fenced";
     if (RETRIABLE_TYPES.has(type)) return "retriable";
     if (POISON_TYPES.has(type)) return "poison";
     if (FATAL_TYPES.has(type)) return "fatal";
@@ -40,9 +41,11 @@ var POISON_TYPES = /* @__PURE__ */ new Set([
   "INVALID_REQUIRED_ACKS",
   "INVALID_PARTITIONS"
 ]);
-var FATAL_TYPES = /* @__PURE__ */ new Set([
+var FENCED_TYPES = /* @__PURE__ */ new Set([
   "INVALID_PRODUCER_EPOCH",
-  "PRODUCER_FENCED",
+  "PRODUCER_FENCED"
+]);
+var FATAL_TYPES = /* @__PURE__ */ new Set([
   "TOPIC_AUTHORIZATION_FAILED",
   "CLUSTER_AUTHORIZATION_FAILED",
   "TRANSACTIONAL_ID_AUTHORIZATION_FAILED",
@@ -73,8 +76,8 @@ var CODE_TO_KIND = /* @__PURE__ */ new Map([
   // TOPIC_AUTHORIZATION_FAILED
   [31, "fatal"],
   // CLUSTER_AUTHORIZATION_FAILED
-  [47, "fatal"],
-  // INVALID_PRODUCER_EPOCH
+  [47, "fenced"],
+  // INVALID_PRODUCER_EPOCH — retryable once via publisher reconnect
   [58, "fatal"],
   // SASL_AUTHENTICATION_FAILED
   [74, "retriable"],
@@ -104,7 +107,13 @@ var UNSUPPORTED_BY_KAFKAJS = [
   "lingerMs",
   "batchSize",
   "deliveryTimeoutMs",
-  "maxRequestSize"
+  "maxRequestSize",
+  // Confluent-only escape hatches; ignored on kafkajs.
+  "compressionLevel",
+  "rawProducerConfig",
+  // librdkafka stats — kafkajs has no equivalent surface.
+  "onStats",
+  "statsIntervalMs"
 ];
 var KafkaJsDriver = class {
   transactional;
@@ -143,13 +152,21 @@ var KafkaJsDriver = class {
       // the provider's returned token (other fields are ignored).
       sasl: this.opts.sasl
     });
-    const createPartitioner = resolveCreatePartitioner(
-      mod.Partitioners,
+    return kafka.producer(await this.buildProducerOptions(mod.Partitioners));
+  }
+  /**
+   * Compute the options object passed to `kafka.producer({...})`. Exposed
+   * as a test seam so power-user escape hatches (customPartitioner,
+   * rawKafkaJsProducerConfig) can be asserted without a live broker.
+   */
+  async buildProducerOptions(partitioners) {
+    const createPartitioner = this.opts.customPartitioner ?? resolveCreatePartitioner(
+      partitioners,
       this.opts.partitioner,
       this.transactional
     );
     const resolvedTxId = this.transactional ? await resolveTransactionalId(this.opts.transactionalId) : void 0;
-    return kafka.producer({
+    return {
       idempotent: this.opts.idempotent ?? true,
       // Idempotent / transactional producers cap maxInFlight at 5. When the
       // user picks transactional we force 1 to keep strict ordering across
@@ -163,13 +180,32 @@ var KafkaJsDriver = class {
       transactionTimeout: this.opts.transactionTimeoutMs,
       // Setting any partitioner choice silences kafkajs's
       // KafkaJSPartitionerNotSpecified warning.
-      createPartitioner
-    });
+      createPartitioner,
+      // Power-user escape hatch — merged LAST so raw keys win against the
+      // translated ones. That's the contract: anything you put here is
+      // final, even if it overrides idempotent/transactionalId/etc.
+      ...this.opts.rawKafkaJsProducerConfig ?? {}
+    };
   }
   async disconnect() {
     await this.producer?.disconnect();
     this.producer = null;
   }
+  /**
+   * Construct a kafkajs admin client wrapped in the eventferry-facing
+   * `KafkaDriverAdmin` shape. The publisher calls `.connect()` on the
+   * returned object before exposing it via `publisher.admin()`.
+   */
+  async admin() {
+    const mod = await importKafkaJs();
+    const kafka = new mod.Kafka({
+      clientId: this.opts.clientId ?? "eventferry-admin",
+      brokers: this.opts.brokers,
+      ssl: this.opts.ssl,
+      sasl: this.opts.sasl
+    });
+    return new KafkaJsAdmin(kafka.admin());
+  }
   async sendBatch(messages) {
     if (!this.producer) throw new Error("KafkaJsDriver not connected");
     const topicMessages = groupByTopic(messages, this.opts.compression);
@@ -258,6 +294,69 @@ function warnUnsupportedKafkajsOptions(opts) {
 function _resetKafkajsWarnDedup() {
   warnedKafkajsKeys.clear();
 }
+var KafkaJsAdmin = class {
+  constructor(client) {
+    this.client = client;
+  }
+  client;
+  async connect() {
+    await this.client.connect();
+  }
+  async close() {
+    await this.client.disconnect();
+  }
+  async listTopics() {
+    return await this.client.listTopics();
+  }
+  async describeTopics(topics) {
+    if (topics.length === 0) return [];
+    const all = new Set(await this.client.listTopics());
+    const existing = topics.filter((t) => all.has(t));
+    const missing = topics.filter((t) => !all.has(t));
+    const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
+    const byName = new Map(meta.topics.map((t) => [t.name, t]));
+    return topics.map((topic) => {
+      if (missing.includes(topic)) return { topic, partitions: [] };
+      const found = byName.get(topic);
+      if (!found) return { topic, partitions: [] };
+      return {
+        topic,
+        partitions: found.partitions.map((p) => ({
+          partitionId: p.partitionId,
+          leader: p.leader,
+          replicas: p.replicas,
+          isr: p.isr
+        }))
+      };
+    });
+  }
+  async createTopics(specs) {
+    if (specs.length === 0) return;
+    const topics = specs.map((s) => ({
+      topic: s.topic,
+      numPartitions: s.numPartitions,
+      replicationFactor: s.replicationFactor,
+      configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
+    }));
+    try {
+      await this.client.createTopics({ topics, waitForLeaders: true });
+    } catch (err) {
+      const e = err;
+      if (e?.type === "TOPIC_ALREADY_EXISTS") return;
+      if (/already exists/i.test(e?.message ?? "")) return;
+      throw err;
+    }
+  }
+  async createPartitions(specs) {
+    if (specs.length === 0) return;
+    await this.client.createPartitions({
+      topicPartitions: specs.map((s) => ({
+        topic: s.topic,
+        count: s.totalCount
+      }))
+    });
+  }
+};
 async function importKafkaJs() {
   try {
     return await import("kafkajs");
@@ -300,8 +399,8 @@ var CODE_TO_KIND2 = /* @__PURE__ */ new Map([
   // ERR__TRANSPORT
   [-198, "poison"],
   // ERR__BAD_COMPRESSION
-  [-144, "fatal"],
-  // ERR__FENCED — producer fenced by another with same txn id
+  [-144, "fenced"],
+  // ERR__FENCED — producer fenced; publisher reconnect attempts a transparent recovery once
   [-150, "fatal"],
   // ERR__FATAL — unrecoverable librdkafka error
   [-169, "fatal"],
@@ -333,8 +432,8 @@ var CODE_TO_KIND2 = /* @__PURE__ */ new Map([
   // TOPIC_AUTHORIZATION_FAILED
   [31, "fatal"],
   // CLUSTER_AUTHORIZATION_FAILED
-  [47, "fatal"],
-  // INVALID_PRODUCER_EPOCH
+  [47, "fenced"],
+  // INVALID_PRODUCER_EPOCH — retryable once via publisher reconnect
   [58, "fatal"],
   // SASL_AUTHENTICATION_FAILED
   [74, "retriable"],
@@ -348,7 +447,7 @@ var CODE_TO_KIND2 = /* @__PURE__ */ new Map([
 ]);
 var NAME_TO_KIND = /* @__PURE__ */ new Map([
   ["ERR__QUEUE_FULL", "backpressure"],
-  ["ERR__FENCED", "fatal"],
+  ["ERR__FENCED", "fenced"],
   ["ERR__FATAL", "fatal"],
   ["ERR__AUTHENTICATION", "fatal"],
   ["ERR__SSL", "fatal"],
@@ -357,7 +456,7 @@ var NAME_TO_KIND = /* @__PURE__ */ new Map([
   ["ERR__BAD_COMPRESSION", "poison"],
   ["ERR_TOPIC_AUTHORIZATION_FAILED", "fatal"],
   ["ERR_CLUSTER_AUTHORIZATION_FAILED", "fatal"],
-  ["ERR_INVALID_PRODUCER_EPOCH", "fatal"],
+  ["ERR_INVALID_PRODUCER_EPOCH", "fenced"],
   ["ERR_SASL_AUTHENTICATION_FAILED", "fatal"],
   ["ERR_CORRUPT_MESSAGE", "poison"],
   ["ERR_MSG_SIZE_TOO_LARGE", "poison"],
@@ -390,6 +489,15 @@ function buildConfluentClientConfig(opts) {
   if (opts.transactionTimeoutMs !== void 0) {
     librdkafka["transaction.timeout.ms"] = opts.transactionTimeoutMs;
   }
+  if (opts.compressionLevel !== void 0) {
+    librdkafka["compression.level"] = opts.compressionLevel;
+  }
+  if (opts.onStats) {
+    librdkafka["stats_cb"] = wrapStatsCallback(opts.onStats);
+    librdkafka["statistics.interval.ms"] = opts.statsIntervalMs ?? 3e4;
+  } else if (opts.statsIntervalMs !== void 0) {
+    librdkafka["statistics.interval.ms"] = opts.statsIntervalMs;
+  }
   const tlsRequested = opts.ssl === true || isTlsConfig(opts.ssl);
   const saslRequested = !!opts.sasl;
   if (saslRequested && tlsRequested) {
@@ -419,11 +527,28 @@ function buildConfluentClientConfig(opts) {
   if (opts.sasl) {
     kafkaJS["sasl"] = opts.sasl;
   }
+  if (opts.rawProducerConfig) {
+    Object.assign(librdkafka, opts.rawProducerConfig);
+  }
   return { kafkaJS, librdkafka };
 }
 function isTlsConfig(v) {
   return typeof v === "object" && v !== null;
 }
+function wrapStatsCallback(onStats) {
+  return (raw) => {
+    let parsed;
+    try {
+      parsed = typeof raw === "string" ? JSON.parse(raw) : raw;
+    } catch {
+      return;
+    }
+    try {
+      onStats(parsed);
+    } catch {
+    }
+  };
+}
 function stringifyPem(input) {
   if (Array.isArray(input)) {
     return input.map((x) => typeof x === "string" ? x : x.toString("utf8")).join("\n");
@@ -472,6 +597,17 @@ var ConfluentDriver = class {
     await this.producer?.disconnect();
     this.producer = null;
   }
+  /**
+   * Construct a librdkafka-backed admin client wrapped in the eventferry
+   * `KafkaDriverAdmin` shape. The publisher's `connect()` is called before
+   * the admin reaches the user.
+   */
+  async admin() {
+    const mod = await importConfluent();
+    const { kafkaJS, librdkafka } = buildConfluentClientConfig(this.opts);
+    const kafka = new mod.KafkaJS.Kafka({ kafkaJS, ...librdkafka });
+    return new ConfluentAdmin(kafka.admin());
+  }
   async sendBatch(messages) {
     if (!this.producer) throw new Error("ConfluentDriver not connected");
     const topicMessages = groupByTopic2(messages);
@@ -540,6 +676,69 @@ function groupByTopic2(messages) {
     messages: msgs
   }));
 }
+var ConfluentAdmin = class {
+  constructor(client) {
+    this.client = client;
+  }
+  client;
+  async connect() {
+    await this.client.connect();
+  }
+  async close() {
+    await this.client.disconnect();
+  }
+  async listTopics() {
+    return await this.client.listTopics();
+  }
+  async describeTopics(topics) {
+    if (topics.length === 0) return [];
+    const all = new Set(await this.client.listTopics());
+    const existing = topics.filter((t) => all.has(t));
+    const missing = topics.filter((t) => !all.has(t));
+    const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
+    const byName = new Map(meta.topics.map((t) => [t.name, t]));
+    return topics.map((topic) => {
+      if (missing.includes(topic)) return { topic, partitions: [] };
+      const found = byName.get(topic);
+      if (!found) return { topic, partitions: [] };
+      return {
+        topic,
+        partitions: found.partitions.map((p) => ({
+          partitionId: p.partitionId,
+          leader: p.leader,
+          replicas: p.replicas,
+          isr: p.isr
+        }))
+      };
+    });
+  }
+  async createTopics(specs) {
+    if (specs.length === 0) return;
+    const topics = specs.map((s) => ({
+      topic: s.topic,
+      numPartitions: s.numPartitions,
+      replicationFactor: s.replicationFactor,
+      configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
+    }));
+    try {
+      await this.client.createTopics({ topics, waitForLeaders: true });
+    } catch (err) {
+      const e = err;
+      if (e?.code === 36 || e?.name === "TOPIC_ALREADY_EXISTS") return;
+      if (/already exists/i.test(e?.message ?? "")) return;
+      throw err;
+    }
+  }
+  async createPartitions(specs) {
+    if (specs.length === 0) return;
+    await this.client.createPartitions({
+      topicPartitions: specs.map((s) => ({
+        topic: s.topic,
+        count: s.totalCount
+      }))
+    });
+  }
+};
 async function importConfluent() {
   try {
     return await import("@confluentinc/kafka-javascript");
@@ -590,10 +789,18 @@ var KafkaPublisher = class {
   logger;
   hooks;
   tracer;
+  validateTopicsOnConnect;
+  autoRecoverFromFence;
+  // Serialize reconnects so concurrent publish() calls hitting a fence
+  // all observe the same single reconnect attempt — the second publish
+  // doesn't try to disconnect a producer the first is still re-initing.
+  fenceRecovery = null;
   constructor(opts) {
     this.logger = opts.logger;
     this.hooks = opts.hooks ?? {};
     this.tracer = opts.tracer ?? new NoopKafkaTracer();
+    this.validateTopicsOnConnect = opts.validateTopicsOnConnect ? Object.freeze([...opts.validateTopicsOnConnect]) : void 0;
+    this.autoRecoverFromFence = opts.autoRecoverFromFence ?? false;
     const onTransactionAbort = this.hooks.onTransactionAbort ? (error) => {
       void safeHook(
         this.logger,
@@ -605,8 +812,90 @@ var KafkaPublisher = class {
   }
   async connect() {
     await this.driver.connect();
+    if (this.validateTopicsOnConnect && this.validateTopicsOnConnect.length) {
+      await this.assertTopicsExist(this.validateTopicsOnConnect);
+    }
     await safeHook(this.logger, "onConnect", () => this.hooks.onConnect?.());
   }
+  /**
+   * Borrow a new admin client from the driver. The returned admin is
+   * connected and ready to use; the CALLER must `close()` it. Throws if the
+   * driver does not implement admin (custom driver lacking the capability).
+   */
+  async admin() {
+    const driverAdmin = await this.openDriverAdmin();
+    return driverAdmin;
+  }
+  /**
+   * Idempotently provision topics. Each spec creates the topic if absent;
+   * existing topics are skipped without error. If `growPartitions: true`
+   * (default false), topics whose current partition count is below the
+   * requested `numPartitions` are grown via `createPartitions`.
+   *
+   * Replication factor and config entries on EXISTING topics are NOT
+   * reconciled — Kafka does not provide a safe in-place alter for those
+   * (changing replication requires reassignment; configs use alterConfigs).
+   * Reach for the raw admin if you need that.
+   */
+  async ensureTopics(specs, opts = {}) {
+    if (specs.length === 0) return;
+    const admin = await this.openDriverAdmin();
+    try {
+      const topicNames = specs.map((s) => s.topic);
+      const existing = await admin.describeTopics(topicNames);
+      const existingByName = new Map(existing.map((t) => [t.topic, t]));
+      const toCreate = specs.filter(
+        (s) => (existingByName.get(s.topic)?.partitions.length ?? 0) === 0
+      );
+      if (toCreate.length) await admin.createTopics(toCreate);
+      if (opts.growPartitions) {
+        const grow = [];
+        for (const s of specs) {
+          if (s.numPartitions === void 0) continue;
+          const current = existingByName.get(s.topic);
+          const currentCount = current?.partitions.length ?? 0;
+          if (currentCount > 0 && currentCount < s.numPartitions) {
+            grow.push({ topic: s.topic, totalCount: s.numPartitions });
+          }
+        }
+        if (grow.length) await admin.createPartitions(grow);
+      }
+    } finally {
+      await admin.close();
+    }
+  }
+  /**
+   * Borrow a fresh admin from the driver and connect it. Throws when the
+   * driver does not implement admin (custom drivers without that capability).
+   */
+  async openDriverAdmin() {
+    if (!this.driver.admin) {
+      throw new Error(
+        "KafkaPublisher: configured driver does not implement admin(). Use the built-in kafkajs or confluent driver, or extend your custom driver."
+      );
+    }
+    const admin = await this.driver.admin();
+    await admin.connect();
+    return admin;
+  }
+  /**
+   * Open an admin, list topics, throw if any required topic is missing.
+   * Always closes the admin (success or failure).
+   */
+  async assertTopicsExist(required) {
+    const admin = await this.openDriverAdmin();
+    try {
+      const all = new Set(await admin.listTopics());
+      const missing = required.filter((t) => !all.has(t));
+      if (missing.length) {
+        throw new Error(
+          `KafkaPublisher: validateTopicsOnConnect failed \u2014 topics missing on cluster: ${missing.join(", ")}`
+        );
+      }
+    } finally {
+      await admin.close();
+    }
+  }
   async disconnect() {
     await this.driver.disconnect();
     await safeHook(
@@ -618,9 +907,14 @@ var KafkaPublisher = class {
   async publish(messages) {
     if (messages.length === 0) return [];
     const span = this.startBatchSpan(messages);
+    const outgoing = this.tracer.inject ? messages.map((m) => {
+      const headers = { ...m.headers };
+      this.tracer.inject(span, headers);
+      return { ...m, headers };
+    }) : messages;
     let results;
     try {
-      results = await this.driver.sendBatch(messages);
+      results = await this.driver.sendBatch(outgoing);
     } catch (err) {
       const error = err instanceof Error ? err : new Error(String(err));
       span.setStatus({ code: "error", message: error.message });
@@ -629,6 +923,20 @@ var KafkaPublisher = class {
       await safeHook(this.logger, "onError", () => this.hooks.onError?.(error));
       throw err;
     }
+    const firstFenced = results.find(
+      (r) => !r.ok && r.errorKind === "fenced"
+    );
+    if (firstFenced) {
+      const fenceErr = firstFenced.error ?? new Error("producer fenced");
+      await safeHook(
+        this.logger,
+        "onProducerFenced",
+        () => this.hooks.onProducerFenced?.(fenceErr)
+      );
+      if (this.autoRecoverFromFence) {
+        results = await this.recoverAndRetry(outgoing, results);
+      }
+    }
     const byId = new Map(messages.map((m) => [m.recordId, m]));
     let allOk = true;
     for (const r of results) {
@@ -679,6 +987,110 @@ var KafkaPublisher = class {
   get transactional() {
     return this.driver.transactional;
   }
+  /**
+   * Cheap reachability probe. Borrows a fresh admin client, calls
+   * `listTopics`, and returns timing + outcome. Useful as the body of a
+   * `/healthz` or `/readyz` endpoint — proves the broker is reachable
+   * AND that the configured credentials still authenticate against it,
+   * without writing a record.
+   *
+   * Does NOT exercise the producer's send path — a healthy admin
+   * connection doesn't guarantee `publish()` will succeed (a fenced
+   * transactional producer would still answer healthy here). Treat this
+   * as "broker reachable + auth still good", not "publisher is fully
+   * operational".
+   *
+   * Default timeout 5_000 ms — long enough to ride out a single broker
+   * leader election, short enough to fail a liveness probe meaningfully.
+   * Set `timeoutMs: 0` to disable the timer entirely.
+   *
+   * The driver must implement `admin()` (the built-ins do); custom
+   * drivers without admin get `{ ok: false, error: ... }` instead of
+   * the throw `publisher.admin()` would surface — health checks are
+   * not the place to crash.
+   */
+  async healthCheck(opts = {}) {
+    const timeoutMs = opts.timeoutMs ?? 5e3;
+    const startedAt = Date.now();
+    if (!this.driver.admin) {
+      return {
+        ok: false,
+        latencyMs: 0,
+        timestamp: startedAt,
+        error: new Error(
+          "KafkaPublisher.healthCheck: configured driver does not implement admin()"
+        )
+      };
+    }
+    let admin = null;
+    try {
+      admin = await this.driver.admin();
+      await admin.connect();
+      const probe = admin.listTopics();
+      if (timeoutMs > 0) {
+        await raceWithTimeout(probe, timeoutMs, "healthCheck");
+      } else {
+        await probe;
+      }
+      return {
+        ok: true,
+        latencyMs: Date.now() - startedAt,
+        timestamp: startedAt
+      };
+    } catch (err) {
+      const error = err instanceof Error ? err : new Error(String(err));
+      return {
+        ok: false,
+        latencyMs: Date.now() - startedAt,
+        timestamp: startedAt,
+        error
+      };
+    } finally {
+      try {
+        await admin?.close();
+      } catch {
+      }
+    }
+  }
+  /**
+   * Disconnect + re-connect the driver and re-send the batch ONCE. Used
+   * by the fence-recovery path. Concurrent fence recoveries dedupe on a
+   * shared in-flight promise (`fenceRecovery`) so we don't tear the
+   * producer down while another batch is mid-restart.
+   *
+   * If the second send STILL reports any fenced records, those failures
+   * are returned unchanged — another instance has almost certainly taken
+   * the same `transactionalId` and silently retrying again would mask
+   * the misconfiguration.
+   */
+  async recoverAndRetry(outgoing, firstResults) {
+    if (!this.fenceRecovery) {
+      this.fenceRecovery = (async () => {
+        try {
+          await this.driver.disconnect();
+          await this.driver.connect();
+        } finally {
+          this.fenceRecovery = null;
+        }
+      })();
+    }
+    try {
+      await this.fenceRecovery;
+    } catch (err) {
+      const reconnectErr = err instanceof Error ? err : new Error(String(err));
+      await safeHook(
+        this.logger,
+        "onError",
+        () => this.hooks.onError?.(reconnectErr)
+      );
+      return firstResults;
+    }
+    try {
+      return await this.driver.sendBatch(outgoing);
+    } catch {
+      return firstResults;
+    }
+  }
   /**
    * Start a span for the batch following the OTel messaging conventions.
    *
@@ -697,6 +1109,26 @@ var KafkaPublisher = class {
     });
   }
 };
+function raceWithTimeout(p, ms, label) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`${label} timed out after ${ms}ms`));
+    }, ms);
+    if (typeof timer.unref === "function") {
+      timer.unref();
+    }
+    p.then(
+      (v) => {
+        clearTimeout(timer);
+        resolve(v);
+      },
+      (e) => {
+        clearTimeout(timer);
+        reject(e);
+      }
+    );
+  });
+}
 function selectDriver(opts) {
   const kind = opts.driver ?? "kafkajs";
   switch (kind) {