@eventferry/kafka 3.4.0 → 3.5.0

package/dist/index.d.cts

@@ -127,7 +127,20 @@ interface KafkaDriver {
  * TLS using the driver's default trust store).
  *
  * `rejectUnauthorized` is intentionally NOT a knob here — TLS verification is
- * non-negotiable. Dev clusters with self-signed certs pass their CA via `ca`.
+ * non-negotiable. Dev clusters with self-signed certs MUST pass their CA via
+ * `ca` (validation still happens, against your CA instead of the system
+ * trust store). If the broker is addressed by an IP literal or a hostname
+ * that doesn't match the cert SAN, set `servername` to the hostname the
+ * cert was issued for so SNI + verification align.
+ *
+ * **Driver parity:**
+ * - `ca`, `cert`, `key`, `passphrase` work on both kafkajs and confluent.
+ * - `servername` is honored by **kafkajs** (Node `tls.connect` reads
+ *   `servername` directly). On the **confluent** driver it's a documented
+ *   no-op — librdkafka derives SNI from the broker address and v1.x's
+ *   kafkaJS-compat layer does not surface an override. Use the kafkajs
+ *   driver for clusters where you need the SNI lever, or wait for
+ *   librdkafka to expose it.
  */
 interface TlsConfig {
     /** PEM-encoded CA bundle. Buffers and strings both accepted. */
@@ -138,7 +151,13 @@ interface TlsConfig {
     key?: string | Buffer;
     /** Passphrase for an encrypted private key. */
     passphrase?: string;
-    /** SNI host. Useful when broker address doesn't match the cert SAN. */
+    /**
+     * SNI host. Set this when the broker address (e.g. an IP literal or an
+     * internal DNS name) does NOT match the certificate's Subject
+     * Alternative Names. Honored on the kafkajs driver; no-op on the
+     * confluent driver (librdkafka does not expose an SNI override at
+     * v1.x).
+     */
     servername?: string;
 }
 /**
@@ -337,7 +356,40 @@ interface ProducerBehaviorConfig {
      * extension point, not a JS callback.
      */
     customPartitioner?: () => (args: unknown) => number;
+    /**
+     * (confluent only) Periodic librdkafka statistics callback. When set,
+     * eventferry wires `stats_cb` on the underlying producer and parses the
+     * JSON payload librdkafka emits every {@link statsIntervalMs} ms.
+     *
+     * The shape is intentionally opaque — librdkafka's stats schema is huge
+     * (txmsgs, rxbytes, queue depth, broker timeouts, per-topic / per-partition
+     * counters…) and evolves across versions. Documented at
+     * https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md.
+     * Cast to your own narrower type if you're consuming a known subset.
+     *
+     * No-op on the kafkajs driver — kafkajs has no equivalent surface.
+     * Pair with {@link statsIntervalMs} (defaults to 30000 ms when this hook
+     * is set but `rawProducerConfig['statistics.interval.ms']` isn't).
+     */
+    onStats?: (stats: LibrdkafkaStats) => void;
+    /**
+     * (confluent only) Override the polling interval the librdkafka stats
+     * callback fires at. Maps to `statistics.interval.ms`. Defaults to
+     * 30000 ms when {@link onStats} is set; defaults to 0 (disabled)
+     * otherwise — librdkafka spends CPU on this and we don't want to enable
+     * it silently. Set to 0 to suppress emission while keeping the hook
+     * defined (useful for tests).
+     */
+    statsIntervalMs?: number;
 }
+/**
+ * Opaque envelope for librdkafka's stats JSON. The schema is
+ * version-specific and large; eventferry surfaces it untyped so you can
+ * cast to whatever subset you care about.
+ *
+ * Reference: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
+ */
+type LibrdkafkaStats = Record<string, unknown>;
 type DriverKind = "kafkajs" | "confluent";
 
 interface KjsProducer {
@@ -522,6 +574,18 @@ interface KafkaPublisherHooks {
      * Useful for observability dashboards that track EOS failure rates.
      */
     onTransactionAbort?(error: Error): void | Promise<void>;
+    /**
+     * Fires when the broker fences this producer — the previous publish
+     * batch reported at least one `errorKind: "fenced"` result. Receives
+     * the first fenced `Error` so dashboards can attribute the incident.
+     *
+     * - When `autoRecoverFromFence` is on, this hook fires BEFORE the
+     *   transparent reconnect attempt. Use it to decrement a transactional
+     *   producer's leader-election counter or warn the operator.
+     * - When `autoRecoverFromFence` is off, the publisher surfaces the
+     *   fenced result unchanged and the hook is still fired for visibility.
+     */
+    onProducerFenced?(error: Error): void | Promise<void>;
 }
 /**
  * Invoke a hook safely. Never throws back into the caller — logs the hook's
@@ -643,6 +707,32 @@ interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorC
      * fire. Driver must implement `admin()` (the built-ins do).
      */
     validateTopicsOnConnect?: string[];
+    /**
+     * Transparently recover from a producer-fence error. When set to `true`,
+     * a `publish()` call whose batch comes back with at least one
+     * `errorKind: "fenced"` result triggers ONE round of:
+     *
+     *   1. disconnect the driver
+     *   2. connect it again (re-running `initTransactions` for transactional producers)
+     *   3. re-send the same batch
+     *
+     * If the second send still produces a fenced result, the publisher gives
+     * up and surfaces the failures unchanged — at that point the fence is
+     * almost certainly caused by another instance taking the same
+     * `transactionalId`, and silently retrying again would mask the
+     * misconfiguration.
+     *
+     * Default `false` to preserve the previous "fenced → fatal" semantics.
+     * Turn it on when running a single producer instance against transient
+     * brokers (rolling restarts, network blips) where a fence is usually
+     * just a transient epoch mismatch.
+     *
+     * For MULTI-INSTANCE EOS, leave this OFF and use a callable
+     * `transactionalId` derived from per-instance context (pod name, k8s
+     * ordinal, AZ + replica index) so each instance has a stable, unique
+     * id — fences will then correctly stop the loser instance.
+     */
+    autoRecoverFromFence?: boolean;
 }
 /**
  * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and adds
@@ -656,6 +746,8 @@ declare class KafkaPublisher implements Publisher {
     private readonly hooks;
     private readonly tracer;
     private readonly validateTopicsOnConnect;
+    private readonly autoRecoverFromFence;
+    private fenceRecovery;
     constructor(opts: KafkaPublisherOptions);
     connect(): Promise<void>;
     /**
@@ -697,6 +789,43 @@ declare class KafkaPublisher implements Publisher {
     publishToDlq(message: PublishableMessage, error: Error): Promise<void>;
     /** Whether the configured driver provides atomic (EOS) batch sends. */
     get transactional(): boolean;
+    /**
+     * Cheap reachability probe. Borrows a fresh admin client, calls
+     * `listTopics`, and returns timing + outcome. Useful as the body of a
+     * `/healthz` or `/readyz` endpoint — proves the broker is reachable
+     * AND that the configured credentials still authenticate against it,
+     * without writing a record.
+     *
+     * Does NOT exercise the producer's send path — a healthy admin
+     * connection doesn't guarantee `publish()` will succeed (a fenced
+     * transactional producer would still answer healthy here). Treat this
+     * as "broker reachable + auth still good", not "publisher is fully
+     * operational".
+     *
+     * Default timeout 5_000 ms — long enough to ride out a single broker
+     * leader election, short enough to fail a liveness probe meaningfully.
+     * Set `timeoutMs: 0` to disable the timer entirely.
+     *
+     * The driver must implement `admin()` (the built-ins do); custom
+     * drivers without admin get `{ ok: false, error: ... }` instead of
+     * the throw `publisher.admin()` would surface — health checks are
+     * not the place to crash.
+     */
+    healthCheck(opts?: {
+        timeoutMs?: number;
+    }): Promise<HealthStatus>;
+    /**
+     * Disconnect + re-connect the driver and re-send the batch ONCE. Used
+     * by the fence-recovery path. Concurrent fence recoveries dedupe on a
+     * shared in-flight promise (`fenceRecovery`) so we don't tear the
+     * producer down while another batch is mid-restart.
+     *
+     * If the second send STILL reports any fenced records, those failures
+     * are returned unchanged — another instance has almost certainly taken
+     * the same `transactionalId` and silently retrying again would mask
+     * the misconfiguration.
+     */
+    private recoverAndRetry;
     /**
      * Start a span for the batch following the OTel messaging conventions.
      *
@@ -707,5 +836,20 @@ declare class KafkaPublisher implements Publisher {
      */
     private startBatchSpan;
 }
+/**
+ * Outcome of a {@link KafkaPublisher.healthCheck} call. Shape is stable
+ * and small so consumers (HTTP /healthz, k8s probes, Datadog) can
+ * marshal it without a translation layer.
+ */
+interface HealthStatus {
+    /** True when the broker answered within the timeout window. */
+    ok: boolean;
+    /** Wall-clock milliseconds spent on the probe (admin connect + listTopics). */
+    latencyMs: number;
+    /** Epoch ms when the probe started — handy for log correlation. */
+    timestamp: number;
+    /** Present only when `ok === false`. The classified error, untouched. */
+    error?: Error;
+}
 
-export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaAdmin, type KafkaConnectionConfig, type KafkaDriver, type KafkaDriverAdmin, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type PartitionGrowSpec, type PartitionMetadata, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, type TopicCreateSpec, type TopicMetadata, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };
+export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type HealthStatus, type KafkaAdmin, type KafkaConnectionConfig, type KafkaDriver, type KafkaDriverAdmin, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, type LibrdkafkaStats, NoopKafkaTracer, type OauthBearerToken, type PartitionGrowSpec, type PartitionMetadata, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, type TopicCreateSpec, type TopicMetadata, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };

package/dist/index.d.ts

@@ -127,7 +127,20 @@ interface KafkaDriver {
  * TLS using the driver's default trust store).
  *
  * `rejectUnauthorized` is intentionally NOT a knob here — TLS verification is
- * non-negotiable. Dev clusters with self-signed certs pass their CA via `ca`.
+ * non-negotiable. Dev clusters with self-signed certs MUST pass their CA via
+ * `ca` (validation still happens, against your CA instead of the system
+ * trust store). If the broker is addressed by an IP literal or a hostname
+ * that doesn't match the cert SAN, set `servername` to the hostname the
+ * cert was issued for so SNI + verification align.
+ *
+ * **Driver parity:**
+ * - `ca`, `cert`, `key`, `passphrase` work on both kafkajs and confluent.
+ * - `servername` is honored by **kafkajs** (Node `tls.connect` reads
+ *   `servername` directly). On the **confluent** driver it's a documented
+ *   no-op — librdkafka derives SNI from the broker address and v1.x's
+ *   kafkaJS-compat layer does not surface an override. Use the kafkajs
+ *   driver for clusters where you need the SNI lever, or wait for
+ *   librdkafka to expose it.
  */
 interface TlsConfig {
     /** PEM-encoded CA bundle. Buffers and strings both accepted. */
@@ -138,7 +151,13 @@ interface TlsConfig {
     key?: string | Buffer;
     /** Passphrase for an encrypted private key. */
     passphrase?: string;
-    /** SNI host. Useful when broker address doesn't match the cert SAN. */
+    /**
+     * SNI host. Set this when the broker address (e.g. an IP literal or an
+     * internal DNS name) does NOT match the certificate's Subject
+     * Alternative Names. Honored on the kafkajs driver; no-op on the
+     * confluent driver (librdkafka does not expose an SNI override at
+     * v1.x).
+     */
     servername?: string;
 }
 /**
@@ -337,7 +356,40 @@ interface ProducerBehaviorConfig {
      * extension point, not a JS callback.
      */
     customPartitioner?: () => (args: unknown) => number;
+    /**
+     * (confluent only) Periodic librdkafka statistics callback. When set,
+     * eventferry wires `stats_cb` on the underlying producer and parses the
+     * JSON payload librdkafka emits every {@link statsIntervalMs} ms.
+     *
+     * The shape is intentionally opaque — librdkafka's stats schema is huge
+     * (txmsgs, rxbytes, queue depth, broker timeouts, per-topic / per-partition
+     * counters…) and evolves across versions. Documented at
+     * https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md.
+     * Cast to your own narrower type if you're consuming a known subset.
+     *
+     * No-op on the kafkajs driver — kafkajs has no equivalent surface.
+     * Pair with {@link statsIntervalMs} (defaults to 30000 ms when this hook
+     * is set but `rawProducerConfig['statistics.interval.ms']` isn't).
+     */
+    onStats?: (stats: LibrdkafkaStats) => void;
+    /**
+     * (confluent only) Override the polling interval the librdkafka stats
+     * callback fires at. Maps to `statistics.interval.ms`. Defaults to
+     * 30000 ms when {@link onStats} is set; defaults to 0 (disabled)
+     * otherwise — librdkafka spends CPU on this and we don't want to enable
+     * it silently. Set to 0 to suppress emission while keeping the hook
+     * defined (useful for tests).
+     */
+    statsIntervalMs?: number;
 }
+/**
+ * Opaque envelope for librdkafka's stats JSON. The schema is
+ * version-specific and large; eventferry surfaces it untyped so you can
+ * cast to whatever subset you care about.
+ *
+ * Reference: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
+ */
+type LibrdkafkaStats = Record<string, unknown>;
 type DriverKind = "kafkajs" | "confluent";
 
 interface KjsProducer {
@@ -522,6 +574,18 @@ interface KafkaPublisherHooks {
      * Useful for observability dashboards that track EOS failure rates.
      */
     onTransactionAbort?(error: Error): void | Promise<void>;
+    /**
+     * Fires when the broker fences this producer — the previous publish
+     * batch reported at least one `errorKind: "fenced"` result. Receives
+     * the first fenced `Error` so dashboards can attribute the incident.
+     *
+     * - When `autoRecoverFromFence` is on, this hook fires BEFORE the
+     *   transparent reconnect attempt. Use it to decrement a transactional
+     *   producer's leader-election counter or warn the operator.
+     * - When `autoRecoverFromFence` is off, the publisher surfaces the
+     *   fenced result unchanged and the hook is still fired for visibility.
+     */
+    onProducerFenced?(error: Error): void | Promise<void>;
 }
 /**
  * Invoke a hook safely. Never throws back into the caller — logs the hook's
@@ -643,6 +707,32 @@ interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorC
      * fire. Driver must implement `admin()` (the built-ins do).
      */
     validateTopicsOnConnect?: string[];
+    /**
+     * Transparently recover from a producer-fence error. When set to `true`,
+     * a `publish()` call whose batch comes back with at least one
+     * `errorKind: "fenced"` result triggers ONE round of:
+     *
+     *   1. disconnect the driver
+     *   2. connect it again (re-running `initTransactions` for transactional producers)
+     *   3. re-send the same batch
+     *
+     * If the second send still produces a fenced result, the publisher gives
+     * up and surfaces the failures unchanged — at that point the fence is
+     * almost certainly caused by another instance taking the same
+     * `transactionalId`, and silently retrying again would mask the
+     * misconfiguration.
+     *
+     * Default `false` to preserve the previous "fenced → fatal" semantics.
+     * Turn it on when running a single producer instance against transient
+     * brokers (rolling restarts, network blips) where a fence is usually
+     * just a transient epoch mismatch.
+     *
+     * For MULTI-INSTANCE EOS, leave this OFF and use a callable
+     * `transactionalId` derived from per-instance context (pod name, k8s
+     * ordinal, AZ + replica index) so each instance has a stable, unique
+     * id — fences will then correctly stop the loser instance.
+     */
+    autoRecoverFromFence?: boolean;
 }
 /**
  * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and adds
@@ -656,6 +746,8 @@ declare class KafkaPublisher implements Publisher {
     private readonly hooks;
     private readonly tracer;
     private readonly validateTopicsOnConnect;
+    private readonly autoRecoverFromFence;
+    private fenceRecovery;
     constructor(opts: KafkaPublisherOptions);
     connect(): Promise<void>;
     /**
@@ -697,6 +789,43 @@ declare class KafkaPublisher implements Publisher {
     publishToDlq(message: PublishableMessage, error: Error): Promise<void>;
     /** Whether the configured driver provides atomic (EOS) batch sends. */
     get transactional(): boolean;
+    /**
+     * Cheap reachability probe. Borrows a fresh admin client, calls
+     * `listTopics`, and returns timing + outcome. Useful as the body of a
+     * `/healthz` or `/readyz` endpoint — proves the broker is reachable
+     * AND that the configured credentials still authenticate against it,
+     * without writing a record.
+     *
+     * Does NOT exercise the producer's send path — a healthy admin
+     * connection doesn't guarantee `publish()` will succeed (a fenced
+     * transactional producer would still answer healthy here). Treat this
+     * as "broker reachable + auth still good", not "publisher is fully
+     * operational".
+     *
+     * Default timeout 5_000 ms — long enough to ride out a single broker
+     * leader election, short enough to fail a liveness probe meaningfully.
+     * Set `timeoutMs: 0` to disable the timer entirely.
+     *
+     * The driver must implement `admin()` (the built-ins do); custom
+     * drivers without admin get `{ ok: false, error: ... }` instead of
+     * the throw `publisher.admin()` would surface — health checks are
+     * not the place to crash.
+     */
+    healthCheck(opts?: {
+        timeoutMs?: number;
+    }): Promise<HealthStatus>;
+    /**
+     * Disconnect + re-connect the driver and re-send the batch ONCE. Used
+     * by the fence-recovery path. Concurrent fence recoveries dedupe on a
+     * shared in-flight promise (`fenceRecovery`) so we don't tear the
+     * producer down while another batch is mid-restart.
+     *
+     * If the second send STILL reports any fenced records, those failures
+     * are returned unchanged — another instance has almost certainly taken
+     * the same `transactionalId` and silently retrying again would mask
+     * the misconfiguration.
+     */
+    private recoverAndRetry;
     /**
      * Start a span for the batch following the OTel messaging conventions.
      *
@@ -707,5 +836,20 @@ declare class KafkaPublisher implements Publisher {
      */
     private startBatchSpan;
 }
+/**
+ * Outcome of a {@link KafkaPublisher.healthCheck} call. Shape is stable
+ * and small so consumers (HTTP /healthz, k8s probes, Datadog) can
+ * marshal it without a translation layer.
+ */
+interface HealthStatus {
+    /** True when the broker answered within the timeout window. */
+    ok: boolean;
+    /** Wall-clock milliseconds spent on the probe (admin connect + listTopics). */
+    latencyMs: number;
+    /** Epoch ms when the probe started — handy for log correlation. */
+    timestamp: number;
+    /** Present only when `ok === false`. The classified error, untouched. */
+    error?: Error;
+}
 
-export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaAdmin, type KafkaConnectionConfig, type KafkaDriver, type KafkaDriverAdmin, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type PartitionGrowSpec, type PartitionMetadata, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, type TopicCreateSpec, type TopicMetadata, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };
+export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type HealthStatus, type KafkaAdmin, type KafkaConnectionConfig, type KafkaDriver, type KafkaDriverAdmin, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, type LibrdkafkaStats, NoopKafkaTracer, type OauthBearerToken, type PartitionGrowSpec, type PartitionMetadata, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, type TopicCreateSpec, type TopicMetadata, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };