@eventferry/kafka 3.3.1 → 3.5.0

package/dist/index.d.cts

@@ -1,5 +1,97 @@
 import { PublishableMessage, PublishResult, Logger, PublishErrorKind, Publisher } from '@eventferry/core';
 
+/**
+ * Admin surface for `KafkaPublisher`. A typed wrapper over each driver's
+ * underlying admin client — implementations live in `kafkajs-driver.ts` and
+ * `confluent-driver.ts`. The publisher exposes this via `publisher.admin()`
+ * (returns a connected admin) and `publisher.ensureTopics()` (idempotent
+ * topic provisioning built on top).
+ *
+ * Scope is deliberately the most-used subset of the Kafka AdminClient
+ * protocol — listing, describing, creating topics and partitions. ACL
+ * management, quota inspection, and consumer-group operations are left to
+ * the underlying client (reach for kafkajs's `Admin` directly if needed).
+ */
+/** Specification for creating one topic. */
+interface TopicCreateSpec {
+    topic: string;
+    /** Default: cluster's `num.partitions` broker setting. */
+    numPartitions?: number;
+    /** Default: cluster's `default.replication.factor` broker setting. */
+    replicationFactor?: number;
+    /**
+     * Per-topic config entries (e.g. `{ "retention.ms": "604800000" }`).
+     * See Kafka broker docs for the full set.
+     */
+    configEntries?: Record<string, string>;
+}
+/** Topic + partition descriptor returned by describeTopics. */
+interface TopicMetadata {
+    topic: string;
+    /** Empty when the topic doesn't exist (so callers can detect absence cheaply). */
+    partitions: PartitionMetadata[];
+}
+interface PartitionMetadata {
+    partitionId: number;
+    /** Broker id of the partition leader; -1 if no leader is known. */
+    leader: number;
+    /** Replica broker ids. */
+    replicas: number[];
+    /** In-sync replica broker ids. */
+    isr: number[];
+}
+/** Specification for growing a topic's partition count (never shrink). */
+interface PartitionGrowSpec {
+    topic: string;
+    /** Total partition count after the change. MUST be ≥ the current count. */
+    totalCount: number;
+}
+/**
+ * Typed admin surface exposed by {@link KafkaPublisher.admin}.
+ *
+ * The returned object is already connected — call `.close()` (or let the
+ * publisher's `disconnect()` cascade close it) when done.
+ *
+ * All methods may throw native errors from the underlying client. The
+ * publisher does not classify admin errors via the `errorKind` machinery
+ * because admin failures are operator-facing and don't flow through the
+ * relay's retry path.
+ */
+interface KafkaAdmin {
+    /** All topic names visible to this principal, including internal topics. */
+    listTopics(): Promise<string[]>;
+    /**
+     * Metadata for the given topics. Topics that don't exist on the cluster
+     * are returned with an empty `partitions` array — callers detect absence
+     * cheaply without a try/catch.
+     */
+    describeTopics(topics: string[]): Promise<TopicMetadata[]>;
+    /**
+     * Create topics. Idempotent at this layer: topics that already exist are
+     * silently skipped (the underlying client may throw `TopicExistsError`;
+     * we swallow it). Use `ensureTopics` on the publisher for partition-count
+     * + replication-factor coherence checks.
+     */
+    createTopics(specs: TopicCreateSpec[]): Promise<void>;
+    /**
+     * Grow each topic's partition count. Never shrinks (Kafka does not
+     * support shrinking partitions). Specs whose `totalCount` equals the
+     * current partition count are silently skipped.
+     */
+    createPartitions(specs: PartitionGrowSpec[]): Promise<void>;
+    /** Disconnect the admin client. */
+    close(): Promise<void>;
+}
+/**
+ * Optional driver-level hook returned by {@link KafkaDriver.admin}. Drivers
+ * implement this thin contract; the publisher composes the higher-level
+ * `KafkaAdmin` and `ensureTopics` on top.
+ */
+interface KafkaDriverAdmin extends KafkaAdmin {
+    /** Called by the publisher before any method is invoked. */
+    connect(): Promise<void>;
+}
+
 /**
  * Low-level driver contract. Each concrete driver (kafkajs, confluent)
  * adapts its native client to this minimal surface. The KafkaPublisher
@@ -19,6 +111,14 @@ interface KafkaDriver {
      * uses this to decide whether `sendBatch` is atomic.
      */
     readonly transactional: boolean;
+    /**
+     * Construct a NEW admin client. The returned admin is not yet connected —
+     * the publisher calls `.connect()` before handing it to the user.
+     *
+     * Optional: drivers without an admin surface may omit this; the publisher
+     * throws a clear error when `publisher.admin()` is called on such a driver.
+     */
+    admin?(): Promise<KafkaDriverAdmin>;
 }
 /**
  * TLS configuration for client connections. Pass a full {@link TlsConfig}
@@ -27,7 +127,20 @@ interface KafkaDriver {
  * TLS using the driver's default trust store).
  *
  * `rejectUnauthorized` is intentionally NOT a knob here — TLS verification is
- * non-negotiable. Dev clusters with self-signed certs pass their CA via `ca`.
+ * non-negotiable. Dev clusters with self-signed certs MUST pass their CA via
+ * `ca` (validation still happens, against your CA instead of the system
+ * trust store). If the broker is addressed by an IP literal or a hostname
+ * that doesn't match the cert SAN, set `servername` to the hostname the
+ * cert was issued for so SNI + verification align.
+ *
+ * **Driver parity:**
+ * - `ca`, `cert`, `key`, `passphrase` work on both kafkajs and confluent.
+ * - `servername` is honored by **kafkajs** (Node `tls.connect` reads
+ *   `servername` directly). On the **confluent** driver it's a documented
+ *   no-op — librdkafka derives SNI from the broker address and v1.x's
+ *   kafkaJS-compat layer does not surface an override. Use the kafkajs
+ *   driver for clusters where you need the SNI lever, or wait for
+ *   librdkafka to expose it.
  */
 interface TlsConfig {
     /** PEM-encoded CA bundle. Buffers and strings both accepted. */
@@ -38,7 +151,13 @@ interface TlsConfig {
     key?: string | Buffer;
     /** Passphrase for an encrypted private key. */
     passphrase?: string;
-    /** SNI host. Useful when broker address doesn't match the cert SAN. */
+    /**
+     * SNI host. Set this when the broker address (e.g. an IP literal or an
+     * internal DNS name) does NOT match the certificate's Subject
+     * Alternative Names. Honored on the kafkajs driver; no-op on the
+     * confluent driver (librdkafka does not expose an SNI override at
+     * v1.x).
+     */
     servername?: string;
 }
 /**
@@ -149,6 +268,14 @@ interface ProducerBehaviorConfig {
     acks?: number;
     /** Compression codec. Driver maps to its native enum. */
     compression?: "none" | "gzip" | "snappy" | "lz4" | "zstd";
+    /**
+     * (confluent only) Compression level for the chosen codec. Defaults vary
+     * per codec — librdkafka picks the broker-friendly default when unset.
+     * Common ranges: gzip 1–9, lz4 0–12, zstd 1–22 (higher = smaller + slower).
+     *
+     * No-op on the kafkajs driver (kafkajs does not expose codec levels).
+     */
+    compressionLevel?: number;
     /**
      * (confluent only) How long the producer waits to accumulate records before
      * flushing a partition batch. Default 0 (ship-immediately). Increase to
@@ -199,7 +326,70 @@ interface ProducerBehaviorConfig {
      * whether this callback throws.
      */
     onTransactionAbort?: (error: Error) => void;
+    /**
+     * (confluent only) Raw librdkafka producer-config keys merged on top of
+     * eventferry's translated config. Use for tuning surface area we don't
+     * expose typed (e.g. `queue.buffering.max.messages`, `socket.keepalive.enable`,
+     * `statistics.interval.ms`). Native keys win against the translated ones,
+     * so this can also be used to override defaults.
+     *
+     * Ignored by the kafkajs driver — log a one-time warning instead of
+     * silently dropping. Use `rawKafkaJsProducerConfig` for kafkajs-side tuning.
+     */
+    rawProducerConfig?: Record<string, unknown>;
+    /**
+     * (kafkajs only) Raw producer-config keys merged into kafkajs's
+     * `kafka.producer({...})` call. Native keys win against the translated
+     * ones. Use for kafkajs-internal knobs like `retry`, `metadataMaxAge`,
+     * `idempotent` overrides, etc.
+     *
+     * No-op on the confluent driver — use `rawProducerConfig` there.
+     */
+    rawKafkaJsProducerConfig?: Record<string, unknown>;
+    /**
+     * (kafkajs only) Custom partitioner factory passed straight to
+     * `kafka.producer({ createPartitioner })`. Overrides {@link partitioner}
+     * preset entirely. See kafkajs docs for the factory signature:
+     * `() => (args: { topic, partitionMetadata, message }) => number`.
+     *
+     * Ignored by the confluent driver — librdkafka's partitioner is a C
+     * extension point, not a JS callback.
+     */
+    customPartitioner?: () => (args: unknown) => number;
+    /**
+     * (confluent only) Periodic librdkafka statistics callback. When set,
+     * eventferry wires `stats_cb` on the underlying producer and parses the
+     * JSON payload librdkafka emits every {@link statsIntervalMs} ms.
+     *
+     * The shape is intentionally opaque — librdkafka's stats schema is huge
+     * (txmsgs, rxbytes, queue depth, broker timeouts, per-topic / per-partition
+     * counters…) and evolves across versions. Documented at
+     * https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md.
+     * Cast to your own narrower type if you're consuming a known subset.
+     *
+     * No-op on the kafkajs driver — kafkajs has no equivalent surface.
+     * Pair with {@link statsIntervalMs} (defaults to 30000 ms when this hook
+     * is set but `rawProducerConfig['statistics.interval.ms']` isn't).
+     */
+    onStats?: (stats: LibrdkafkaStats) => void;
+    /**
+     * (confluent only) Override the polling interval the librdkafka stats
+     * callback fires at. Maps to `statistics.interval.ms`. Defaults to
+     * 30000 ms when {@link onStats} is set; defaults to 0 (disabled)
+     * otherwise — librdkafka spends CPU on this and we don't want to enable
+     * it silently. Set to 0 to suppress emission while keeping the hook
+     * defined (useful for tests).
+     */
+    statsIntervalMs?: number;
 }
+/**
+ * Opaque envelope for librdkafka's stats JSON. The schema is
+ * version-specific and large; eventferry surfaces it untyped so you can
+ * cast to whatever subset you care about.
+ *
+ * Reference: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
+ */
+type LibrdkafkaStats = Record<string, unknown>;
 type DriverKind = "kafkajs" | "confluent";
 
 interface KjsProducer {
@@ -213,6 +403,11 @@ interface KjsTransaction {
     commit(): Promise<void>;
     abort(): Promise<void>;
 }
+interface KjsPartitionersNamespace {
+    DefaultPartitioner: () => unknown;
+    LegacyPartitioner: () => unknown;
+    JavaCompatiblePartitioner: () => unknown;
+}
 interface KafkaJsDriverOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
     /**
      * Optional logger for the driver's own diagnostics (e.g. warnings about
@@ -235,7 +430,19 @@ declare class KafkaJsDriver implements KafkaDriver {
      * the send/transaction logic can be exercised without a real broker.
      */
     protected createProducer(): Promise<KjsProducer>;
+    /**
+     * Compute the options object passed to `kafka.producer({...})`. Exposed
+     * as a test seam so power-user escape hatches (customPartitioner,
+     * rawKafkaJsProducerConfig) can be asserted without a live broker.
+     */
+    protected buildProducerOptions(partitioners: KjsPartitionersNamespace | undefined): Promise<Record<string, unknown>>;
     disconnect(): Promise<void>;
+    /**
+     * Construct a kafkajs admin client wrapped in the eventferry-facing
+     * `KafkaDriverAdmin` shape. The publisher calls `.connect()` on the
+     * returned object before exposing it via `publisher.admin()`.
+     */
+    admin(): Promise<KafkaDriverAdmin>;
     sendBatch(messages: PublishableMessage[]): Promise<PublishResult[]>;
 }
 /** Internal — used by tests. Resets the dedup so warnings can be observed in isolation. */
@@ -288,6 +495,12 @@ declare class ConfluentDriver implements KafkaDriver {
      */
     protected createProducer(): Promise<CkProducer>;
     disconnect(): Promise<void>;
+    /**
+     * Construct a librdkafka-backed admin client wrapped in the eventferry
+     * `KafkaDriverAdmin` shape. The publisher's `connect()` is called before
+     * the admin reaches the user.
+     */
+    admin(): Promise<KafkaDriverAdmin>;
     sendBatch(messages: PublishableMessage[]): Promise<PublishResult[]>;
 }
 
@@ -361,6 +574,18 @@ interface KafkaPublisherHooks {
      * Useful for observability dashboards that track EOS failure rates.
      */
     onTransactionAbort?(error: Error): void | Promise<void>;
+    /**
+     * Fires when the broker fences this producer — the previous publish
+     * batch reported at least one `errorKind: "fenced"` result. Receives
+     * the first fenced `Error` so dashboards can attribute the incident.
+     *
+     * - When `autoRecoverFromFence` is on, this hook fires BEFORE the
+     *   transparent reconnect attempt. Use it to decrement a transactional
+     *   producer's leader-election counter or warn the operator.
+     * - When `autoRecoverFromFence` is off, the publisher surfaces the
+     *   fenced result unchanged and the hook is still fired for visibility.
+     */
+    onProducerFenced?(error: Error): void | Promise<void>;
 }
 /**
  * Invoke a hook safely. Never throws back into the caller — logs the hook's
@@ -423,6 +648,21 @@ interface KafkaTracer {
      *                    server.address/port).
      */
     startPublishSpan(name: string, attributes: Record<string, SpanAttributeValue>): SpanLike;
+    /**
+     * OPTIONAL: inject the active trace context (W3C `traceparent` +
+     * `tracestate`) into a per-message header map. Called by the publisher
+     * AFTER the batch span is created and BEFORE the records hit the wire.
+     *
+     * Implementations typically wrap OpenTelemetry's `propagation.inject(...)`
+     * or your tracing SDK's equivalent. Mutate the `headers` object in
+     * place — the publisher allocates a fresh copy per message so this is
+     * safe and matches the propagation API of every major SDK.
+     *
+     * Tracers without distributed-context propagation (or that only care
+     * about local spans) may leave this off — consumers can still derive
+     * trace headers themselves by other means.
+     */
+    inject?(span: SpanLike, headers: Record<string, string>): void;
 }
 /**
  * No-op tracer. Used when the user does not configure one. Cheap allocation
@@ -457,6 +697,42 @@ interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorC
      * Use a thin adapter over your tracing SDK (see {@link KafkaTracer}).
      */
     tracer?: KafkaTracer;
+    /**
+     * If set, `connect()` checks that every topic in this list exists on the
+     * cluster and throws a descriptive error if any are missing. Use this to
+     * fail-fast at startup instead of letting the first send-time error
+     * surprise you.
+     *
+     * Validation runs AFTER the producer connects but BEFORE `onConnect` hooks
+     * fire. Driver must implement `admin()` (the built-ins do).
+     */
+    validateTopicsOnConnect?: string[];
+    /**
+     * Transparently recover from a producer-fence error. When set to `true`,
+     * a `publish()` call whose batch comes back with at least one
+     * `errorKind: "fenced"` result triggers ONE round of:
+     *
+     *   1. disconnect the driver
+     *   2. connect it again (re-running `initTransactions` for transactional producers)
+     *   3. re-send the same batch
+     *
+     * If the second send still produces a fenced result, the publisher gives
+     * up and surfaces the failures unchanged — at that point the fence is
+     * almost certainly caused by another instance taking the same
+     * `transactionalId`, and silently retrying again would mask the
+     * misconfiguration.
+     *
+     * Default `false` to preserve the previous "fenced → fatal" semantics.
+     * Turn it on when running a single producer instance against transient
+     * brokers (rolling restarts, network blips) where a fence is usually
+     * just a transient epoch mismatch.
+     *
+     * For MULTI-INSTANCE EOS, leave this OFF and use a callable
+     * `transactionalId` derived from per-instance context (pod name, k8s
+     * ordinal, AZ + replica index) so each instance has a stable, unique
+     * id — fences will then correctly stop the loser instance.
+     */
+    autoRecoverFromFence?: boolean;
 }
 /**
  * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and adds
@@ -469,8 +745,41 @@ declare class KafkaPublisher implements Publisher {
     private readonly logger;
     private readonly hooks;
     private readonly tracer;
+    private readonly validateTopicsOnConnect;
+    private readonly autoRecoverFromFence;
+    private fenceRecovery;
     constructor(opts: KafkaPublisherOptions);
     connect(): Promise<void>;
+    /**
+     * Borrow a new admin client from the driver. The returned admin is
+     * connected and ready to use; the CALLER must `close()` it. Throws if the
+     * driver does not implement admin (custom driver lacking the capability).
+     */
+    admin(): Promise<KafkaAdmin>;
+    /**
+     * Idempotently provision topics. Each spec creates the topic if absent;
+     * existing topics are skipped without error. If `growPartitions: true`
+     * (default false), topics whose current partition count is below the
+     * requested `numPartitions` are grown via `createPartitions`.
+     *
+     * Replication factor and config entries on EXISTING topics are NOT
+     * reconciled — Kafka does not provide a safe in-place alter for those
+     * (changing replication requires reassignment; configs use alterConfigs).
+     * Reach for the raw admin if you need that.
+     */
+    ensureTopics(specs: TopicCreateSpec[], opts?: {
+        growPartitions?: boolean;
+    }): Promise<void>;
+    /**
+     * Borrow a fresh admin from the driver and connect it. Throws when the
+     * driver does not implement admin (custom drivers without that capability).
+     */
+    private openDriverAdmin;
+    /**
+     * Open an admin, list topics, throw if any required topic is missing.
+     * Always closes the admin (success or failure).
+     */
+    private assertTopicsExist;
     disconnect(): Promise<void>;
     publish(messages: PublishableMessage[]): Promise<PublishResult[]>;
     /**
@@ -480,6 +789,43 @@ declare class KafkaPublisher implements Publisher {
     publishToDlq(message: PublishableMessage, error: Error): Promise<void>;
     /** Whether the configured driver provides atomic (EOS) batch sends. */
     get transactional(): boolean;
+    /**
+     * Cheap reachability probe. Borrows a fresh admin client, calls
+     * `listTopics`, and returns timing + outcome. Useful as the body of a
+     * `/healthz` or `/readyz` endpoint — proves the broker is reachable
+     * AND that the configured credentials still authenticate against it,
+     * without writing a record.
+     *
+     * Does NOT exercise the producer's send path — a healthy admin
+     * connection doesn't guarantee `publish()` will succeed (a fenced
+     * transactional producer would still answer healthy here). Treat this
+     * as "broker reachable + auth still good", not "publisher is fully
+     * operational".
+     *
+     * Default timeout 5_000 ms — long enough to ride out a single broker
+     * leader election, short enough to fail a liveness probe meaningfully.
+     * Set `timeoutMs: 0` to disable the timer entirely.
+     *
+     * The driver must implement `admin()` (the built-ins do); custom
+     * drivers without admin get `{ ok: false, error: ... }` instead of
+     * the throw `publisher.admin()` would surface — health checks are
+     * not the place to crash.
+     */
+    healthCheck(opts?: {
+        timeoutMs?: number;
+    }): Promise<HealthStatus>;
+    /**
+     * Disconnect + re-connect the driver and re-send the batch ONCE. Used
+     * by the fence-recovery path. Concurrent fence recoveries dedupe on a
+     * shared in-flight promise (`fenceRecovery`) so we don't tear the
+     * producer down while another batch is mid-restart.
+     *
+     * If the second send STILL reports any fenced records, those failures
+     * are returned unchanged — another instance has almost certainly taken
+     * the same `transactionalId` and silently retrying again would mask
+     * the misconfiguration.
+     */
+    private recoverAndRetry;
     /**
      * Start a span for the batch following the OTel messaging conventions.
      *
@@ -490,5 +836,20 @@ declare class KafkaPublisher implements Publisher {
      */
     private startBatchSpan;
 }
+/**
+ * Outcome of a {@link KafkaPublisher.healthCheck} call. Shape is stable
+ * and small so consumers (HTTP /healthz, k8s probes, Datadog) can
+ * marshal it without a translation layer.
+ */
+interface HealthStatus {
+    /** True when the broker answered within the timeout window. */
+    ok: boolean;
+    /** Wall-clock milliseconds spent on the probe (admin connect + listTopics). */
+    latencyMs: number;
+    /** Epoch ms when the probe started — handy for log correlation. */
+    timestamp: number;
+    /** Present only when `ok === false`. The classified error, untouched. */
+    error?: Error;
+}
 
-export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };
+export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type HealthStatus, type KafkaAdmin, type KafkaConnectionConfig, type KafkaDriver, type KafkaDriverAdmin, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, type LibrdkafkaStats, NoopKafkaTracer, type OauthBearerToken, type PartitionGrowSpec, type PartitionMetadata, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, type TopicCreateSpec, type TopicMetadata, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };