@eventferry/kafka 3.1.0 → 3.2.0

package/dist/index.d.cts

@@ -1,4 +1,4 @@
-import { PublishableMessage, PublishResult, PublishErrorKind, Publisher } from '@eventferry/core';
+import { PublishableMessage, PublishResult, Logger, PublishErrorKind, Publisher } from '@eventferry/core';
 
 /**
  * Low-level driver contract. Each concrete driver (kafkajs, confluent)
@@ -130,8 +130,21 @@ interface ProducerBehaviorConfig {
      * Requires a stable transactionalId. Default false.
      */
     transactional?: boolean;
-    /** Required when transactional=true. Must be stable per producer instance. */
-    transactionalId?: string;
+    /**
+     * Required when `transactional=true`. Must be stable per producer instance
+     * — two producers sharing the same id race for the broker-side epoch and
+     * fence each other.
+     *
+     * Accepts a string OR a thunk that resolves the id at connect time. The
+     * callable form lets you derive the id from runtime context that's not
+     * known at construction (pod name, AZ + replica index, k8s ordinal):
+     *
+     *     transactionalId: () => `${process.env.POD_NAME}-${replicaIndex()}`,
+     *
+     * For multi-instance EOS, the derived id MUST be stable across a single
+     * instance's restarts but UNIQUE across instances.
+     */
+    transactionalId?: string | (() => string | Promise<string>);
     /** acks: -1/"all" (default), 0, or 1. */
     acks?: number;
     /** Compression codec. Driver maps to its native enum. */
@@ -176,6 +189,16 @@ interface ProducerBehaviorConfig {
      * silences kafkajs's `KafkaJSPartitionerNotSpecified` warning.
      */
     partitioner?: KafkaJsPartitionerChoice;
+    /**
+     * Callback fired when a transactional `sendBatch` triggers the abort
+     * path (e.g. mid-batch driver error, broker rejection). Used by the
+     * publisher to fan out the matching `KafkaPublisherHooks.onTransactionAbort`
+     * hook — but advanced users constructing a driver directly may also wire
+     * it themselves. Best-effort: the driver still proceeds to abort the
+     * underlying transaction and return per-record failures regardless of
+     * whether this callback throws.
+     */
+    onTransactionAbort?: (error: Error) => void;
 }
 type DriverKind = "kafkajs" | "confluent";
 
@@ -191,6 +214,12 @@ interface KjsTransaction {
     abort(): Promise<void>;
 }
 interface KafkaJsDriverOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
+    /**
+     * Optional logger for the driver's own diagnostics (e.g. warnings about
+     * unsupported tuning options). When absent the driver falls back to
+     * `console.warn` so existing users see the same output.
+     */
+    logger?: Logger;
 }
 /**
  * Driver backed by the pure-JS `kafkajs` client. Simple, zero native deps.
@@ -299,6 +328,110 @@ interface ConfluentClientConfig {
 }
 declare function buildConfluentClientConfig(opts: KafkaConnectionConfig & ProducerBehaviorConfig): ConfluentClientConfig;
 
+/**
+ * Lifecycle hooks fired by `KafkaPublisher`. Every hook is optional. The
+ * publisher wraps each invocation in a try/catch and logs (via the
+ * configured logger) on failure — a misbehaving hook will NEVER break
+ * publishing.
+ *
+ * Typical wiring:
+ *   - Custom observability stacks (Datadog APM, New Relic) → `onPublish`,
+ *     `onError`, `onTransactionAbort`.
+ *   - Connection-aware readiness probes → `onConnect` / `onDisconnect`.
+ *   - Audit logs of every published record → `onPublish`.
+ */
+interface KafkaPublisherHooks {
+    /** Fires after the underlying client successfully connects. */
+    onConnect?(): void | Promise<void>;
+    /** Fires after the underlying client disconnects (clean shutdown). */
+    onDisconnect?(): void | Promise<void>;
+    /**
+     * Fires once per record after a publish attempt — both successes and
+     * failures. The `result.ok` flag distinguishes them.
+     */
+    onPublish?(result: PublishResult, message: PublishableMessage): void | Promise<void>;
+    /**
+     * Fires for any error surfaced from the publish path — driver-thrown
+     * errors, transaction abort errors, etc. `message` is set when the error
+     * is per-record; absent for batch-level errors (e.g. connect failure).
+     */
+    onError?(error: Error, message?: PublishableMessage): void | Promise<void>;
+    /**
+     * Fires when a transactional sendBatch's inner abort path is taken.
+     * Useful for observability dashboards that track EOS failure rates.
+     */
+    onTransactionAbort?(error: Error): void | Promise<void>;
+}
+/**
+ * Invoke a hook safely. Never throws back into the caller — logs the hook's
+ * failure via the configured logger (or no-op when logger is absent).
+ */
+declare function safeHook(logger: Logger | undefined, hookName: keyof KafkaPublisherHooks, invoke: () => void | Promise<void> | undefined): Promise<void>;
+
+/**
+ * Tracing surface for the publisher.
+ *
+ * eventferry deliberately does not depend on `@opentelemetry/api` — instead
+ * users wire a thin adapter over their tracing system (OpenTelemetry,
+ * Datadog, internal, …). This file defines the minimal contract; an
+ * OpenTelemetry adapter is ~10 lines (see the README).
+ *
+ * The contract follows the **current stable** OpenTelemetry messaging
+ * semantic conventions
+ * ({@link https://github.com/open-telemetry/semantic-conventions/blob/main/docs/messaging/kafka.md spec}):
+ *
+ *   - Span name: `"{topic} publish"`
+ *   - `SpanKind.PRODUCER`
+ *   - Required attributes: `messaging.system=kafka`,
+ *     `messaging.operation.type=publish`, `messaging.destination.name=<topic>`
+ *   - Recommended: `messaging.batch.message_count`, `messaging.kafka.partition`,
+ *     `server.address`, `server.port`
+ *   - One span per batch (NOT per message — per-message spans cause
+ *     cardinality explosion and the spec actively warns against this)
+ */
+/** Attribute values the spec allows. */
+type SpanAttributeValue = string | number | boolean;
+/**
+ * Minimal span surface the publisher needs. Implementations wrap a
+ * tracing-system-specific span; methods MUST never throw out of the
+ * publisher's hot path (wrap your own SDK calls in try/catch).
+ */
+interface SpanLike {
+    setAttribute(key: string, value: SpanAttributeValue): void;
+    setAttributes(attrs: Record<string, SpanAttributeValue>): void;
+    /** OK on success; ERROR on failure. The `message` is the error message. */
+    setStatus(status: {
+        code: "ok" | "error";
+        message?: string;
+    }): void;
+    /** Attach an exception to the span (OpenTelemetry `recordException`). */
+    recordException(error: Error): void;
+    end(): void;
+}
+/**
+ * Factory the publisher calls once per `sendBatch` to start a span.
+ * Implementations MUST set `SpanKind.PRODUCER` and the messaging semconv
+ * attributes on the returned span before returning it.
+ */
+interface KafkaTracer {
+    /**
+     * Start a publish span.
+     * @param name        Recommended format: `"{topic} publish"`.
+     * @param attributes  Initial attributes (the publisher supplies the messaging
+     *                    semconv set: system, destination.name, operation.type,
+     *                    batch.message_count, plus optional kafka.partition and
+     *                    server.address/port).
+     */
+    startPublishSpan(name: string, attributes: Record<string, SpanAttributeValue>): SpanLike;
+}
+/**
+ * No-op tracer. Used when the user does not configure one. Cheap allocation
+ * — never touches I/O.
+ */
+declare class NoopKafkaTracer implements KafkaTracer {
+    startPublishSpan(): SpanLike;
+}
+
 interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
     /** Which underlying client to use. Default "kafkajs". */
     driver?: DriverKind;
@@ -307,14 +440,35 @@ interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorC
      * Useful for testing or unsupported clients.
      */
     customDriver?: KafkaDriver;
+    /**
+     * Optional structured logger. When set, the publisher routes its own
+     * diagnostics (driver warnings, hook failures) through it. When omitted,
+     * the publisher is silent — only the underlying drivers may still log.
+     */
+    logger?: Logger;
+    /**
+     * Optional lifecycle hooks. Every hook is invoked safely (try/catch +
+     * logged via `logger`) and a misbehaving hook will never break publishing.
+     */
+    hooks?: KafkaPublisherHooks;
+    /**
+     * Optional tracer. When set, `publish()` wraps each batch in a span that
+     * follows the current stable OpenTelemetry messaging semantic conventions.
+     * Use a thin adapter over your tracing SDK (see {@link KafkaTracer}).
+     */
+    tracer?: KafkaTracer;
 }
 /**
- * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and
- * adds dead-letter routing. Works against Kafka and Redpanda identically
- * (Redpanda is Kafka-API compatible).
+ * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and adds
+ * dead-letter routing, observability hooks, and OpenTelemetry-shaped publish
+ * spans. Works against Kafka and Redpanda identically (Redpanda is Kafka-API
+ * compatible).
  */
 declare class KafkaPublisher implements Publisher {
     private readonly driver;
+    private readonly logger;
+    private readonly hooks;
+    private readonly tracer;
     constructor(opts: KafkaPublisherOptions);
     connect(): Promise<void>;
     disconnect(): Promise<void>;
@@ -326,6 +480,15 @@ declare class KafkaPublisher implements Publisher {
     publishToDlq(message: PublishableMessage, error: Error): Promise<void>;
     /** Whether the configured driver provides atomic (EOS) batch sends. */
     get transactional(): boolean;
+    /**
+     * Start a span for the batch following the OTel messaging conventions.
+     *
+     * Multi-topic batches: per the OTel spec, the span name uses the
+     * destination — we pick the FIRST topic in the batch and document the
+     * limitation. Callers that publish heterogeneous batches and care about
+     * per-topic spans should split their batches upstream.
+     */
+    private startBatchSpan;
 }
 
-export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherOptions, type OauthBearerToken, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type TlsConfig, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError };
+export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { PublishableMessage, PublishResult, PublishErrorKind, Publisher } from '@eventferry/core';
+import { PublishableMessage, PublishResult, Logger, PublishErrorKind, Publisher } from '@eventferry/core';
 
 /**
  * Low-level driver contract. Each concrete driver (kafkajs, confluent)
@@ -130,8 +130,21 @@ interface ProducerBehaviorConfig {
      * Requires a stable transactionalId. Default false.
      */
     transactional?: boolean;
-    /** Required when transactional=true. Must be stable per producer instance. */
-    transactionalId?: string;
+    /**
+     * Required when `transactional=true`. Must be stable per producer instance
+     * — two producers sharing the same id race for the broker-side epoch and
+     * fence each other.
+     *
+     * Accepts a string OR a thunk that resolves the id at connect time. The
+     * callable form lets you derive the id from runtime context that's not
+     * known at construction (pod name, AZ + replica index, k8s ordinal):
+     *
+     *     transactionalId: () => `${process.env.POD_NAME}-${replicaIndex()}`,
+     *
+     * For multi-instance EOS, the derived id MUST be stable across a single
+     * instance's restarts but UNIQUE across instances.
+     */
+    transactionalId?: string | (() => string | Promise<string>);
     /** acks: -1/"all" (default), 0, or 1. */
     acks?: number;
     /** Compression codec. Driver maps to its native enum. */
@@ -176,6 +189,16 @@ interface ProducerBehaviorConfig {
      * silences kafkajs's `KafkaJSPartitionerNotSpecified` warning.
      */
     partitioner?: KafkaJsPartitionerChoice;
+    /**
+     * Callback fired when a transactional `sendBatch` triggers the abort
+     * path (e.g. mid-batch driver error, broker rejection). Used by the
+     * publisher to fan out the matching `KafkaPublisherHooks.onTransactionAbort`
+     * hook — but advanced users constructing a driver directly may also wire
+     * it themselves. Best-effort: the driver still proceeds to abort the
+     * underlying transaction and return per-record failures regardless of
+     * whether this callback throws.
+     */
+    onTransactionAbort?: (error: Error) => void;
 }
 type DriverKind = "kafkajs" | "confluent";
 
@@ -191,6 +214,12 @@ interface KjsTransaction {
     abort(): Promise<void>;
 }
 interface KafkaJsDriverOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
+    /**
+     * Optional logger for the driver's own diagnostics (e.g. warnings about
+     * unsupported tuning options). When absent the driver falls back to
+     * `console.warn` so existing users see the same output.
+     */
+    logger?: Logger;
 }
 /**
  * Driver backed by the pure-JS `kafkajs` client. Simple, zero native deps.
@@ -299,6 +328,110 @@ interface ConfluentClientConfig {
 }
 declare function buildConfluentClientConfig(opts: KafkaConnectionConfig & ProducerBehaviorConfig): ConfluentClientConfig;
 
+/**
+ * Lifecycle hooks fired by `KafkaPublisher`. Every hook is optional. The
+ * publisher wraps each invocation in a try/catch and logs (via the
+ * configured logger) on failure — a misbehaving hook will NEVER break
+ * publishing.
+ *
+ * Typical wiring:
+ *   - Custom observability stacks (Datadog APM, New Relic) → `onPublish`,
+ *     `onError`, `onTransactionAbort`.
+ *   - Connection-aware readiness probes → `onConnect` / `onDisconnect`.
+ *   - Audit logs of every published record → `onPublish`.
+ */
+interface KafkaPublisherHooks {
+    /** Fires after the underlying client successfully connects. */
+    onConnect?(): void | Promise<void>;
+    /** Fires after the underlying client disconnects (clean shutdown). */
+    onDisconnect?(): void | Promise<void>;
+    /**
+     * Fires once per record after a publish attempt — both successes and
+     * failures. The `result.ok` flag distinguishes them.
+     */
+    onPublish?(result: PublishResult, message: PublishableMessage): void | Promise<void>;
+    /**
+     * Fires for any error surfaced from the publish path — driver-thrown
+     * errors, transaction abort errors, etc. `message` is set when the error
+     * is per-record; absent for batch-level errors (e.g. connect failure).
+     */
+    onError?(error: Error, message?: PublishableMessage): void | Promise<void>;
+    /**
+     * Fires when a transactional sendBatch's inner abort path is taken.
+     * Useful for observability dashboards that track EOS failure rates.
+     */
+    onTransactionAbort?(error: Error): void | Promise<void>;
+}
+/**
+ * Invoke a hook safely. Never throws back into the caller — logs the hook's
+ * failure via the configured logger (or no-op when logger is absent).
+ */
+declare function safeHook(logger: Logger | undefined, hookName: keyof KafkaPublisherHooks, invoke: () => void | Promise<void> | undefined): Promise<void>;
+
+/**
+ * Tracing surface for the publisher.
+ *
+ * eventferry deliberately does not depend on `@opentelemetry/api` — instead
+ * users wire a thin adapter over their tracing system (OpenTelemetry,
+ * Datadog, internal, …). This file defines the minimal contract; an
+ * OpenTelemetry adapter is ~10 lines (see the README).
+ *
+ * The contract follows the **current stable** OpenTelemetry messaging
+ * semantic conventions
+ * ({@link https://github.com/open-telemetry/semantic-conventions/blob/main/docs/messaging/kafka.md spec}):
+ *
+ *   - Span name: `"{topic} publish"`
+ *   - `SpanKind.PRODUCER`
+ *   - Required attributes: `messaging.system=kafka`,
+ *     `messaging.operation.type=publish`, `messaging.destination.name=<topic>`
+ *   - Recommended: `messaging.batch.message_count`, `messaging.kafka.partition`,
+ *     `server.address`, `server.port`
+ *   - One span per batch (NOT per message — per-message spans cause
+ *     cardinality explosion and the spec actively warns against this)
+ */
+/** Attribute values the spec allows. */
+type SpanAttributeValue = string | number | boolean;
+/**
+ * Minimal span surface the publisher needs. Implementations wrap a
+ * tracing-system-specific span; methods MUST never throw out of the
+ * publisher's hot path (wrap your own SDK calls in try/catch).
+ */
+interface SpanLike {
+    setAttribute(key: string, value: SpanAttributeValue): void;
+    setAttributes(attrs: Record<string, SpanAttributeValue>): void;
+    /** OK on success; ERROR on failure. The `message` is the error message. */
+    setStatus(status: {
+        code: "ok" | "error";
+        message?: string;
+    }): void;
+    /** Attach an exception to the span (OpenTelemetry `recordException`). */
+    recordException(error: Error): void;
+    end(): void;
+}
+/**
+ * Factory the publisher calls once per `sendBatch` to start a span.
+ * Implementations MUST set `SpanKind.PRODUCER` and the messaging semconv
+ * attributes on the returned span before returning it.
+ */
+interface KafkaTracer {
+    /**
+     * Start a publish span.
+     * @param name        Recommended format: `"{topic} publish"`.
+     * @param attributes  Initial attributes (the publisher supplies the messaging
+     *                    semconv set: system, destination.name, operation.type,
+     *                    batch.message_count, plus optional kafka.partition and
+     *                    server.address/port).
+     */
+    startPublishSpan(name: string, attributes: Record<string, SpanAttributeValue>): SpanLike;
+}
+/**
+ * No-op tracer. Used when the user does not configure one. Cheap allocation
+ * — never touches I/O.
+ */
+declare class NoopKafkaTracer implements KafkaTracer {
+    startPublishSpan(): SpanLike;
+}
+
 interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
     /** Which underlying client to use. Default "kafkajs". */
     driver?: DriverKind;
@@ -307,14 +440,35 @@ interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorC
      * Useful for testing or unsupported clients.
      */
     customDriver?: KafkaDriver;
+    /**
+     * Optional structured logger. When set, the publisher routes its own
+     * diagnostics (driver warnings, hook failures) through it. When omitted,
+     * the publisher is silent — only the underlying drivers may still log.
+     */
+    logger?: Logger;
+    /**
+     * Optional lifecycle hooks. Every hook is invoked safely (try/catch +
+     * logged via `logger`) and a misbehaving hook will never break publishing.
+     */
+    hooks?: KafkaPublisherHooks;
+    /**
+     * Optional tracer. When set, `publish()` wraps each batch in a span that
+     * follows the current stable OpenTelemetry messaging semantic conventions.
+     * Use a thin adapter over your tracing SDK (see {@link KafkaTracer}).
+     */
+    tracer?: KafkaTracer;
 }
 /**
- * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and
- * adds dead-letter routing. Works against Kafka and Redpanda identically
- * (Redpanda is Kafka-API compatible).
+ * The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and adds
+ * dead-letter routing, observability hooks, and OpenTelemetry-shaped publish
+ * spans. Works against Kafka and Redpanda identically (Redpanda is Kafka-API
+ * compatible).
  */
 declare class KafkaPublisher implements Publisher {
     private readonly driver;
+    private readonly logger;
+    private readonly hooks;
+    private readonly tracer;
     constructor(opts: KafkaPublisherOptions);
     connect(): Promise<void>;
     disconnect(): Promise<void>;
@@ -326,6 +480,15 @@ declare class KafkaPublisher implements Publisher {
     publishToDlq(message: PublishableMessage, error: Error): Promise<void>;
     /** Whether the configured driver provides atomic (EOS) batch sends. */
     get transactional(): boolean;
+    /**
+     * Start a span for the batch following the OTel messaging conventions.
+     *
+     * Multi-topic batches: per the OTel spec, the span name uses the
+     * destination — we pick the FIRST topic in the batch and document the
+     * limitation. Callers that publish heterogeneous batches and care about
+     * per-topic spans should split their batches upstream.
+     */
+    private startBatchSpan;
 }
 
-export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherOptions, type OauthBearerToken, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type TlsConfig, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError };
+export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };