@eventferry/kafka 3.0.0 → 3.2.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { PublishableMessage, PublishResult, PublishErrorKind, Publisher } from '@eventferry/core';
+import { PublishableMessage, PublishResult, Logger, PublishErrorKind, Publisher } from '@eventferry/core';
 
 /**
  * Low-level driver contract. Each concrete driver (kafkajs, confluent)
@@ -20,17 +20,108 @@ interface KafkaDriver {
      */
     readonly transactional: boolean;
 }
+/**
+ * TLS configuration for client connections. Pass a full {@link TlsConfig}
+ * when the cluster requires CA pinning, mutual TLS (client cert + key), or a
+ * specific SNI host. Plain `ssl: true` keeps the previous behavior (one-way
+ * TLS using the driver's default trust store).
+ *
+ * `rejectUnauthorized` is intentionally NOT a knob here — TLS verification is
+ * non-negotiable. Dev clusters with self-signed certs pass their CA via `ca`.
+ */
+interface TlsConfig {
+    /** PEM-encoded CA bundle. Buffers and strings both accepted. */
+    ca?: string | Buffer | Array<string | Buffer>;
+    /** PEM-encoded client certificate (required for mTLS). */
+    cert?: string | Buffer;
+    /** PEM-encoded private key for the client certificate (required for mTLS). */
+    key?: string | Buffer;
+    /** Passphrase for an encrypted private key. */
+    passphrase?: string;
+    /** SNI host. Useful when broker address doesn't match the cert SAN. */
+    servername?: string;
+}
+/**
+ * Username + password SASL: PLAIN and SCRAM-SHA-256/512. The conventional
+ * "API key + secret" shape used by Confluent Cloud, Aiven, on-prem SCRAM.
+ */
+interface SaslPasswordConfig {
+    mechanism: "plain" | "scram-sha-256" | "scram-sha-512";
+    username: string;
+    password: string;
+}
+/**
+ * Token returned by an OAUTHBEARER provider.
+ *
+ * Driver asymmetry (verified against `kafkajs/types/index.d.ts` and
+ * `@confluentinc/kafka-javascript/types/kafkajs.d.ts`):
+ *
+ * - `kafkajs` reads only `value`. Other fields are silently ignored.
+ * - `@confluentinc/kafka-javascript` REQUIRES `value` + `principal` + `lifetime`
+ *   and accepts an optional `extensions` map. Passing only `{ value }` throws.
+ *
+ * Cross-driver portable providers MUST populate all four. eventferry treats
+ * `principal` / `lifetime` / `extensions` as optional in the type to support
+ * kafkajs-only setups; supplying them is a no-op there.
+ */
+interface OauthBearerToken {
+    /** The bearer token string (JWT, opaque, …). */
+    value: string;
+    /** Principal name. REQUIRED on the confluent driver. */
+    principal?: string;
+    /** Lifetime in MILLISECONDS. REQUIRED on the confluent driver. */
+    lifetime?: number;
+    /** SASL extensions to send alongside the token (e.g. for OIDC scopes). */
+    extensions?: Record<string, string>;
+}
+/**
+ * SASL/OAUTHBEARER: bring-your-own token provider. The function is invoked
+ * by the underlying client on demand (NOT on a fixed timer); cache the
+ * token in your provider if you want to amortise issuance cost.
+ *
+ * Required for Azure Event Hubs, Confluent Cloud with OAuth/SSO, and any
+ * OIDC-fronted Kafka. For AWS MSK IAM, wrap the AWS SigV4 signer in this
+ * callback.
+ */
+interface SaslOauthbearerConfig {
+    mechanism: "oauthbearer";
+    oauthBearerProvider: () => Promise<OauthBearerToken>;
+}
+/**
+ * Discriminated union over the SASL mechanisms eventferry supports today.
+ * Add new mechanisms by extending this union and mapping them in each driver.
+ */
+type SaslConfig = SaslPasswordConfig | SaslOauthbearerConfig;
 /** Shared connection config accepted by both drivers. */
 interface KafkaConnectionConfig {
     brokers: string[];
     clientId?: string;
-    ssl?: boolean;
-    sasl?: {
-        mechanism: "plain" | "scram-sha-256" | "scram-sha-512";
-        username: string;
-        password: string;
-    };
+    /**
+     * TLS configuration. `true` enables one-way TLS using the driver's default
+     * trust store; a {@link TlsConfig} object lets you supply a custom CA,
+     * client cert (for mTLS), and SNI host.
+     */
+    ssl?: boolean | TlsConfig;
+    sasl?: SaslConfig;
 }
+/**
+ * Choice of partitioner. Only honored by the kafkajs driver — the confluent
+ * driver uses librdkafka's `consistent_random` (key-aware sticky) and
+ * partitioner override is out of scope for this release.
+ *
+ * - `"java-compatible"` (recommended for greenfield): kafkajs's
+ *   `Partitioners.JavaCompatiblePartitioner`. Matches the Java client's
+ *   murmur2-based hash so producers across language boundaries land on the
+ *   same partition for the same key.
+ * - `"legacy"`: kafkajs's pre-v2 partitioner. Use when migrating an existing
+ *   topic where hash continuity matters.
+ * - `"default"`: kafkajs's current default. Equivalent to legacy in v2 but
+ *   may change with major kafkajs releases.
+ *
+ * Setting this also silences the noisy `KafkaJSPartitionerNotSpecified`
+ * warning kafkajs emits when no partitioner choice is made explicitly.
+ */
+type KafkaJsPartitionerChoice = "default" | "legacy" | "java-compatible";
 interface ProducerBehaviorConfig {
     /** Enable idempotent producer (dedup + ordering). Default true. */
     idempotent?: boolean;
@@ -39,12 +130,75 @@ interface ProducerBehaviorConfig {
      * Requires a stable transactionalId. Default false.
      */
     transactional?: boolean;
-    /** Required when transactional=true. Must be stable per producer instance. */
-    transactionalId?: string;
+    /**
+     * Required when `transactional=true`. Must be stable per producer instance
+     * — two producers sharing the same id race for the broker-side epoch and
+     * fence each other.
+     *
+     * Accepts a string OR a thunk that resolves the id at connect time. The
+     * callable form lets you derive the id from runtime context that's not
+     * known at construction (pod name, AZ + replica index, k8s ordinal):
+     *
+     *     transactionalId: () => `${process.env.POD_NAME}-${replicaIndex()}`,
+     *
+     * For multi-instance EOS, the derived id MUST be stable across a single
+     * instance's restarts but UNIQUE across instances.
+     */
+    transactionalId?: string | (() => string | Promise<string>);
     /** acks: -1/"all" (default), 0, or 1. */
     acks?: number;
     /** Compression codec. Driver maps to its native enum. */
     compression?: "none" | "gzip" | "snappy" | "lz4" | "zstd";
+    /**
+     * (confluent only) How long the producer waits to accumulate records before
+     * flushing a partition batch. Default 0 (ship-immediately). Increase to
+     * 10–50ms for higher throughput at the cost of latency.
+     */
+    lingerMs?: number;
+    /** (confluent only) Maximum bytes per partition batch before forced flush. */
+    batchSize?: number;
+    /**
+     * Max concurrent unacknowledged producer requests. MUST be ≤5 when
+     * `idempotent: true`. Higher = throughput; lower = stricter ordering on
+     * non-idempotent producers (no other path preserves order on retry).
+     */
+    maxInFlightRequests?: number;
+    /** Per-request broker-ack timeout. Default 30 s. */
+    requestTimeoutMs?: number;
+    /**
+     * (confluent only) End-to-end timeout for a record from produce() call to
+     * terminal success / failure (includes retries). Defaults to 120 s.
+     * If this exceeds the relay's `claimTimeoutMs`, the reaper may double-
+     * publish a slow record — set both coherently.
+     */
+    deliveryTimeoutMs?: number;
+    /**
+     * (confluent only) Max bytes of a single record (after compression).
+     * MUST be ≤ broker's `message.max.bytes`. Defaults to 1 MB.
+     */
+    maxRequestSize?: number;
+    /**
+     * Broker-side ceiling on how long a transaction can stay open before
+     * auto-abort. Maps to `transaction.timeout.ms`. Default 60 s; capped by
+     * the broker's `transaction.max.timeout.ms`.
+     */
+    transactionTimeoutMs?: number;
+    /**
+     * (kafkajs only) Choice of partitioner. See
+     * {@link KafkaJsPartitionerChoice} for the options. Setting any value
+     * silences kafkajs's `KafkaJSPartitionerNotSpecified` warning.
+     */
+    partitioner?: KafkaJsPartitionerChoice;
+    /**
+     * Callback fired when a transactional `sendBatch` triggers the abort
+     * path (e.g. mid-batch driver error, broker rejection). Used by the
+     * publisher to fan out the matching `KafkaPublisherHooks.onTransactionAbort`
+     * hook — but advanced users constructing a driver directly may also wire
+     * it themselves. Best-effort: the driver still proceeds to abort the
+     * underlying transaction and return per-record failures regardless of
+     * whether this callback throws.
+     */
+    onTransactionAbort?: (error: Error) => void;
 }
 type DriverKind = "kafkajs" | "confluent";
 
@@ -60,6 +214,12 @@ interface KjsTransaction {
     abort(): Promise<void>;
 }
 interface KafkaJsDriverOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
+    /**
+     * Optional logger for the driver's own diagnostics (e.g. warnings about
+     * unsupported tuning options). When absent the driver falls back to
+     * `console.warn` so existing users see the same output.
+     */
+    logger?: Logger;
 }
 /**
  * Driver backed by the pure-JS `kafkajs` client. Simple, zero native deps.
@@ -78,6 +238,8 @@ declare class KafkaJsDriver implements KafkaDriver {
     disconnect(): Promise<void>;
     sendBatch(messages: PublishableMessage[]): Promise<PublishResult[]>;
 }
+/** Internal — used by tests. Resets the dedup so warnings can be observed in isolation. */
+declare function _resetKafkajsWarnDedup(): void;
 
 /**
  * Classify a kafkajs producer error into a {@link PublishErrorKind} so the
@@ -144,6 +306,132 @@ declare class ConfluentDriver implements KafkaDriver {
  */
 declare function classifyConfluentError(err: unknown): PublishErrorKind;
 
+/**
+ * Translate eventferry's normalized `KafkaConnectionConfig` into the shape
+ * expected by `@confluentinc/kafka-javascript`'s `KafkaJS.Kafka` constructor.
+ *
+ * Returns an object with two parts:
+ *   - `kafkaJS`: the kafkajs-compatible config layer (clientId, brokers, and
+ *     simple ssl/sasl when no advanced TLS is needed).
+ *   - top-level keys: librdkafka-style config (e.g. `ssl.ca.pem`,
+ *     `security.protocol`) used when the user supplies a {@link TlsConfig}.
+ *
+ * Why a separate translator: the kafkajs-compat layer accepts the simple
+ * `ssl: true` boolean but the verified path for mTLS (CA + cert + key) is
+ * librdkafka's `ssl.*.pem` keys. The translator picks the right surface
+ * based on what the caller supplied. Buffer inputs are coerced to strings —
+ * librdkafka accepts PEM strings, NOT Buffers.
+ */
+interface ConfluentClientConfig {
+    kafkaJS: Record<string, unknown>;
+    librdkafka: Record<string, unknown>;
+}
+declare function buildConfluentClientConfig(opts: KafkaConnectionConfig & ProducerBehaviorConfig): ConfluentClientConfig;
+
+/**
+ * Lifecycle hooks fired by `KafkaPublisher`. Every hook is optional. The
+ * publisher wraps each invocation in a try/catch and logs (via the
+ * configured logger) on failure — a misbehaving hook will NEVER break
+ * publishing.
+ *
+ * Typical wiring:
+ *   - Custom observability stacks (Datadog APM, New Relic) → `onPublish`,
+ *     `onError`, `onTransactionAbort`.
+ *   - Connection-aware readiness probes → `onConnect` / `onDisconnect`.
+ *   - Audit logs of every published record → `onPublish`.
+ */
+interface KafkaPublisherHooks {
+    /** Fires after the underlying client successfully connects. */
+    onConnect?(): void | Promise<void>;
+    /** Fires after the underlying client disconnects (clean shutdown). */
+    onDisconnect?(): void | Promise<void>;
+    /**
+     * Fires once per record after a publish attempt — both successes and
+     * failures. The `result.ok` flag distinguishes them.
+     */
+    onPublish?(result: PublishResult, message: PublishableMessage): void | Promise<void>;
+    /**
+     * Fires for any error surfaced from the publish path — driver-thrown
+     * errors, transaction abort errors, etc. `message` is set when the error
+     * is per-record; absent for batch-level errors (e.g. connect failure).
+     */
+    onError?(error: Error, message?: PublishableMessage): void | Promise<void>;
+    /**
+     * Fires when a transactional sendBatch's inner abort path is taken.
+     * Useful for observability dashboards that track EOS failure rates.
+     */
+    onTransactionAbort?(error: Error): void | Promise<void>;
+}
+/**
+ * Invoke a hook safely. Never throws back into the caller — logs the hook's
+ * failure via the configured logger (or no-op when logger is absent).
+ */
+declare function safeHook(logger: Logger | undefined, hookName: keyof KafkaPublisherHooks, invoke: () => void | Promise<void> | undefined): Promise<void>;
+
+/**
+ * Tracing surface for the publisher.
+ *
+ * eventferry deliberately does not depend on `@opentelemetry/api` — instead
+ * users wire a thin adapter over their tracing system (OpenTelemetry,
+ * Datadog, internal, …). This file defines the minimal contract; an
+ * OpenTelemetry adapter is ~10 lines (see the README).
+ *
+ * The contract follows the **current stable** OpenTelemetry messaging
+ * semantic conventions
+ * ({@link https://github.com/open-telemetry/semantic-conventions/blob/main/docs/messaging/kafka.md spec}):
+ *
+ *   - Span name: `"{topic} publish"`
+ *   - `SpanKind.PRODUCER`
+ *   - Required attributes: `messaging.system=kafka`,
+ *     `messaging.operation.type=publish`, `messaging.destination.name=<topic>`
+ *   - Recommended: `messaging.batch.message_count`, `messaging.kafka.partition`,
+ *     `server.address`, `server.port`
+ *   - One span per batch (NOT per message — per-message spans cause
+ *     cardinality explosion and the spec actively warns against this)
+ */
+/** Attribute values the spec allows. */
+type SpanAttributeValue = string | number | boolean;
+/**
+ * Minimal span surface the publisher needs. Implementations wrap a
+ * tracing-system-specific span; methods MUST never throw out of the
+ * publisher's hot path (wrap your own SDK calls in try/catch).
+ */
+interface SpanLike {
+    setAttribute(key: string, value: SpanAttributeValue): void;
+    setAttributes(attrs: Record<string, SpanAttributeValue>): void;
+    /** OK on success; ERROR on failure. The `message` is the error message. */
+    setStatus(status: {
+        code: "ok" | "error";
+        message?: string;
+    }): void;
+    /** Attach an exception to the span (OpenTelemetry `recordException`). */
+    recordException(error: Error): void;
+    end(): void;
+}
+/**
+ * Factory the publisher calls once per `sendBatch` to start a span.
+ * Implementations MUST set `SpanKind.PRODUCER` and the messaging semconv
+ * attributes on the returned span before returning it.
+ */
+interface KafkaTracer {
+    /**
+     * Start a publish span.
+     * @param name        Recommended format: `"{topic} publish"`.
+     * @param attributes  Initial attributes (the publisher supplies the messaging
+     *                    semconv set: system, destination.name, operation.type,
+     *                    batch.message_count, plus optional kafka.partition and
+     *                    server.address/port).
+     */
+    startPublishSpan(name: string, attributes: Record<string, SpanAttributeValue>): SpanLike;
+}
+/**
+ * No-op tracer. Used when the user does not configure one. Cheap allocation
+ * — never touches I/O.
+ */
+declare class NoopKafkaTracer implements KafkaTracer {
+    startPublishSpan(): SpanLike;
+}
+
 interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
     /** Which underlying client to use. Default "kafkajs". */
     driver?: DriverKind;
@@ -152,14 +440,35 @@ interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorC
      * Useful for testing or unsupported clients.
      */
     customDriver?: KafkaDriver;
+    /**
+     * Optional structured logger. When set, the publisher routes its own
+     * diagnostics (driver warnings, hook failures) through it. When omitted,
+     * the publisher is silent — only the underlying drivers may still log.
+     */
+    logger?: Logger;
+    /**
+     * Optional lifecycle hooks. Every hook is invoked safely (try/catch +
+     * logged via `logger`) and a misbehaving hook will never break publishing.
+     */
+    hooks?: KafkaPublisherHooks;
+    /**
+     * Optional tracer. When set, `publish()` wraps each batch in a span that
+     * follows the current stable OpenTelemetry messaging semantic conventions.
+     * Use a thin adapter over your tracing SDK (see {@link KafkaTracer}).
+     */
+    tracer?: KafkaTracer;
 }
 /**
- * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and
- * adds dead-letter routing. Works against Kafka and Redpanda identically
- * (Redpanda is Kafka-API compatible).
+ * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and adds
+ * dead-letter routing, observability hooks, and OpenTelemetry-shaped publish
+ * spans. Works against Kafka and Redpanda identically (Redpanda is Kafka-API
+ * compatible).
  */
 declare class KafkaPublisher implements Publisher {
     private readonly driver;
+    private readonly logger;
+    private readonly hooks;
+    private readonly tracer;
     constructor(opts: KafkaPublisherOptions);
     connect(): Promise<void>;
     disconnect(): Promise<void>;
@@ -171,6 +480,15 @@ declare class KafkaPublisher implements Publisher {
     publishToDlq(message: PublishableMessage, error: Error): Promise<void>;
     /** Whether the configured driver provides atomic (EOS) batch sends. */
     get transactional(): boolean;
+    /**
+     * Start a span for the batch following the OTel messaging conventions.
+     *
+     * Multi-topic batches: per the OTel spec, the span name uses the
+     * destination — we pick the FIRST topic in the batch and document the
+     * limitation. Callers that publish heterogeneous batches and care about
+     * per-topic spans should split their batches upstream.
+     */
+    private startBatchSpan;
 }
 
-export { ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, KafkaPublisher, type KafkaPublisherOptions, type ProducerBehaviorConfig, classifyConfluentError, classifyKafkajsError };
+export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };