actor-ts 0.9.0

package/dist/io/broker/KafkaActor.d.ts

@@ -0,0 +1,316 @@
+import type { Config } from '../../config/Config.js';
+import type { ActorRef } from '../../ActorRef.js';
+import { BrokerActor, type OutboundEnvelope } from './BrokerActor.js';
+import type { BrokerCommonSettings } from './BrokerSettings.js';
+/** Inbound Kafka record delivered to subscribers. */
+export interface KafkaRecord {
+    readonly topic: string;
+    readonly partition: number;
+    readonly offset: string;
+    readonly key: Uint8Array | null;
+    readonly value: Uint8Array | null;
+    readonly timestamp: string;
+    readonly headers: Readonly<Record<string, Uint8Array | string | null>>;
+}
+/** Outbound Kafka publish envelope.  `key` / `partition` optional. */
+export interface KafkaPublish {
+    readonly topic: string;
+    readonly value: Uint8Array | string;
+    readonly key?: Uint8Array | string;
+    readonly partition?: number;
+    readonly headers?: Readonly<Record<string, string | Uint8Array>>;
+}
+/**
+ * Offset-commit policy (#2):
+ *
+ *   - `'auto'` — kafkajs commits after the message handler returns
+ *     successfully (the default).  At-least-once delivery: the actor
+ *     might re-deliver a message if it crashes between handler and
+ *     commit, so handlers must be idempotent.
+ *   - `'manual'` — the consumer pump pauses on each message and
+ *     waits for an explicit `{ kind: 'commit', topic, partition,
+ *     offset }` command before resolving the kafkajs promise and
+ *     advancing the partition.  Combined with idempotent producing,
+ *     this gives "exactly-once-with-processing" semantics.
+ */
+export type KafkaCommitMode = 'auto' | 'manual';
+export interface KafkaActorSettings extends BrokerCommonSettings {
+    /** Bootstrap servers (`'kafka-1:9092,kafka-2:9092'` or array). */
+    readonly brokers?: ReadonlyArray<string> | string;
+    /** Stable client id reported to the broker. */
+    readonly clientId?: string;
+    /** Optional SASL credentials. */
+    readonly sasl?: {
+        readonly mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
+        readonly username: string;
+        readonly password: string;
+    };
+    /** Enable TLS. */
+    readonly ssl?: boolean;
+    /** Producer settings. */
+    readonly producer?: {
+        readonly idempotent?: boolean;
+        readonly allowAutoTopicCreation?: boolean;
+    };
+    /** Consumer settings.  `groupId` is required to start a consumer. */
+    readonly consumer?: {
+        readonly groupId?: string;
+        readonly fromBeginning?: boolean;
+        /**
+         * Offset-commit policy.  Default `'auto'` — kafkajs auto-commits
+         * after the handler returns (at-least-once).  See
+         * {@link KafkaCommitMode} for the `'manual'` (exactly-once-with-
+         * processing) shape.
+         */
+        readonly commitMode?: KafkaCommitMode;
+        /**
+         * Max time in ms the manual-commit pump waits for an external
+         * `commit` / `nack` before giving up on a message and letting
+         * kafkajs reject it (which triggers a rebalance and re-delivery).
+         * Only used when `commitMode === 'manual'`.  Default 30s.
+         */
+        readonly commitTimeoutMs?: number;
+    };
+    /** Subscriber that receives every consumed record. */
+    readonly target?: ActorRef<KafkaRecord>;
+    /** Topics the consumer subscribes to at connect time. */
+    readonly topics?: ReadonlyArray<string>;
+}
+export type KafkaCmd = {
+    readonly kind: 'publish';
+    readonly publish: KafkaPublish;
+} | {
+    readonly kind: 'subscribe';
+    readonly topic: string;
+}
+/**
+ * Commit the offset for a message that was delivered in
+ * `commitMode: 'manual'` mode.  The pump's `eachMessage` promise
+ * resolves; kafkajs commits `offset + 1` and reads the next message
+ * on this partition.  Sending `commit` outside manual-commit mode
+ * is silently ignored.
+ */
+ | {
+    readonly kind: 'commit';
+    readonly topic: string;
+    readonly partition: number;
+    readonly offset: string;
+}
+/**
+ * Negative-acknowledge a manual-commit message — the offset is
+ * **not** committed and the eachMessage promise rejects, so
+ * kafkajs treats the partition as having failed: on rebalance /
+ * restart the same offset will be re-delivered.  The optional
+ * `reason` shows up in the actor's warn log.
+ */
+ | {
+    readonly kind: 'nack';
+    readonly topic: string;
+    readonly partition: number;
+    readonly offset: string;
+    readonly reason?: string;
+}
+/**
+ * Bump the consumer's session-deadline mid-processing (#78).
+ * `commitMode: 'manual'` pauses the eachMessage pump until the
+ * handler ack's; if the handler runs longer than the consumer's
+ * `sessionTimeoutMs` (kafkajs default 30 s) the broker evicts the
+ * member, the partition rebalances, and the message is
+ * re-delivered after the rebalance settles.
+ *
+ * `heartbeat` invokes the captured kafkajs `heartbeat()` callback
+ * for the still-pending record, which restarts the session clock
+ * without touching the offset.  Send it periodically from any
+ * handler that's likely to exceed `sessionTimeoutMs / 3`; the
+ * `withAutoHeartbeat()` helper schedules it for you.  Heartbeats
+ * for an unknown / already-committed record are a silent no-op.
+ */
+ | {
+    readonly kind: 'heartbeat';
+    readonly topic: string;
+    readonly partition: number;
+    readonly offset: string;
+};
+/**
+ * Kafka producer + consumer in one actor, backed by `kafkajs`.  When
+ * `consumer.groupId` is set, a consumer is started after `connectImpl`
+ * and consumed records are delivered to `target`.  When a producer is
+ * the only goal, leave `consumer` and `topics` empty.
+ *
+ * **Offset-commit semantics.**
+ *
+ *   - `commitMode: 'auto'` (default) — kafkajs commits after each
+ *     handler returns successfully → **at-least-once**.  Cheap; OK
+ *     for idempotent handlers.
+ *   - `commitMode: 'manual'` — pump pauses on each message and waits
+ *     for an explicit `commit` command from the handler (#2).  The
+ *     handler is responsible for sending exactly one `commit` (or
+ *     `nack`) per delivered record.  If neither arrives within
+ *     `commitTimeoutMs` the pump rejects internally, kafkajs treats
+ *     the partition as failed, and re-delivery happens on rebalance.
+ *     Produces **exactly-once-with-processing**: a message that
+ *     successfully passed through `commit` is committed; a crash or
+ *     `nack` re-delivers.
+ *
+ *   const kafka = system.spawnAnonymous(Props.create(() => new KafkaActor({
+ *     brokers: ['kafka:9092'],
+ *     consumer: { groupId: 'orders', commitMode: 'manual' },
+ *     topics: ['orders'],
+ *     target: orderProcessor,
+ *   })));
+ *
+ *   class OrderProcessor extends Actor<KafkaRecord> {
+ *     constructor(private readonly kafka: ActorRef<KafkaCmd>) { super(); }
+ *     async onReceive(rec: KafkaRecord) {
+ *       try {
+ *         await db.insertOrder(JSON.parse(rec.value!.toString()));
+ *         this.kafka.tell({ kind: 'commit', topic: rec.topic,
+ *                            partition: rec.partition, offset: rec.offset });
+ *       } catch (e) {
+ *         this.kafka.tell({ kind: 'nack',   topic: rec.topic,
+ *                            partition: rec.partition, offset: rec.offset });
+ *       }
+ *     }
+ *   }
+ */
+export declare class KafkaActor extends BrokerActor<KafkaActorSettings, KafkaCmd, KafkaPublish> {
+    private kafka;
+    private producer;
+    private consumer;
+    /**
+     * Map of `<topic>|<partition>|<offset>` → in-flight commit promise
+     * resolver.  Only populated in `commitMode: 'manual'`; entries are
+     * inserted by the eachMessage pump and removed by `commit` /
+     * `nack` / the timeout.
+     */
+    private readonly pendingCommits;
+    constructor(settings?: Partial<KafkaActorSettings>);
+    protected configKey(): string;
+    protected builtInDefaults(): Partial<KafkaActorSettings>;
+    protected readSettingsFromConfig(c: Config): Partial<KafkaActorSettings>;
+    protected requiredSettings(): ReadonlyArray<keyof KafkaActorSettings>;
+    protected endpointLabel(): string;
+    /**
+     * Build a `KafkaInstanceLike` from the configured settings.  Override
+     * in a subclass for tests that want to inject mock producers /
+     * consumers without going through the kafkajs peer dep — that's the
+     * test seam used by `tests/unit/io/broker/KafkaActor.test.ts`.
+     */
+    protected createKafkaInstance(): Promise<KafkaInstanceLike>;
+    protected connectImpl(): Promise<void>;
+    protected disconnectImpl(): Promise<void>;
+    protected dispatchOutgoing(env: OutboundEnvelope<KafkaPublish>): Promise<void>;
+    onReceive(cmd: KafkaCmd): void;
+    private handleCommit;
+    /**
+     * Forward a `heartbeat` command to the captured kafkajs callback —
+     * keeps the record's session-deadline alive without affecting the
+     * offset.  Unknown record (auto-mode, already-committed, expired)
+     * is a silent no-op so handlers can call this unconditionally
+     * without checking `commitMode` or whether the record's commit has
+     * already landed elsewhere.
+     */
+    private handleHeartbeat;
+    private handleNack;
+}
+/**
+ * Minimal Kafka surface the actor depends on.  Exported so test seams
+ * (subclasses overriding `createKafkaInstance`) can satisfy the same
+ * shape without pulling kafkajs.
+ */
+export interface KafkaInstanceLike {
+    producer(config?: {
+        idempotent?: boolean;
+        allowAutoTopicCreation?: boolean;
+    }): KafkaProducerLike;
+    consumer(config: {
+        groupId: string;
+    }): KafkaConsumerLike;
+}
+export interface KafkaProducerLike {
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    send(args: {
+        topic: string;
+        messages: Array<{
+            value: Uint8Array | null;
+            key?: Uint8Array | null;
+            partition?: number;
+            headers?: Record<string, string | Uint8Array>;
+        }>;
+    }): Promise<unknown>;
+}
+interface KafkaConsumedMessage {
+    topic: string;
+    partition: number;
+    message: {
+        offset: string;
+        key: Uint8Array | null;
+        value: Uint8Array | null;
+        timestamp: string;
+        headers?: Record<string, Uint8Array | string | null>;
+    };
+    /** kafkajs callback for explicit heartbeats during long handler runs. */
+    heartbeat?: () => Promise<void>;
+}
+export interface KafkaConsumerLike {
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    subscribe(args: {
+        topic: string;
+        fromBeginning?: boolean;
+    }): Promise<void>;
+    run(args: {
+        autoCommit?: boolean;
+        eachMessage: (m: KafkaConsumedMessage) => Promise<void>;
+    }): Promise<void>;
+    /**
+     * Manual offset commit (#2).  Each entry's `offset` is the **next**
+     * message to consume — i.e., one past the message we just finished
+     * processing.  Required for `commitMode: 'manual'`; older kafkajs
+     * releases that lack it surface a clear error at commit time.
+     */
+    commitOffsets?(args: ReadonlyArray<{
+        topic: string;
+        partition: number;
+        offset: string;
+    }>): Promise<void>;
+}
+/**
+ * Reference to a Kafka record for use with {@link withAutoHeartbeat} —
+ * exactly the (topic, partition, offset) triple a `KafkaCmd.heartbeat`
+ * command needs.  `KafkaRecord` itself satisfies this shape, so the
+ * `eachMessage` payload from the consumer pump can be passed directly.
+ */
+export interface KafkaRecordRef {
+    readonly topic: string;
+    readonly partition: number;
+    readonly offset: string;
+}
+/**
+ * Run `body` while periodically telling `kafka` to heartbeat the
+ * record.  Cancels the timer on success or failure; the body's
+ * resolution / rejection is propagated.  Use this when a manual-commit
+ * handler is likely to exceed the consumer's `sessionTimeoutMs / 3` —
+ * the rule of thumb being three heartbeats per session window so a
+ * single dropped tick doesn't trigger eviction.
+ *
+ * Default `everyMs` is 10 s, suitable for kafkajs's default 30 s
+ * `sessionTimeout`.  If you tuned the consumer's session timeout up,
+ * scale `everyMs` to match.
+ *
+ * @example
+ * ```ts
+ * await withAutoHeartbeat({ kafka, record: rec, everyMs: 5_000 }, async () => {
+ *   await reallySlowDatabaseWork(rec.value);
+ * });
+ * kafka.tell({ kind: 'commit', ...rec });
+ * ```
+ */
+export declare function withAutoHeartbeat<T>(args: {
+    readonly kafka: ActorRef<KafkaCmd>;
+    readonly record: KafkaRecordRef;
+    readonly everyMs?: number;
+}, body: () => Promise<T>): Promise<T>;
+export {};
+//# sourceMappingURL=KafkaActor.d.ts.map

package/dist/io/broker/KafkaActor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"KafkaActor.d.ts","sourceRoot":"","sources":["../../../src/io/broker/KafkaActor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAGlD,OAAO,EAAE,WAAW,EAAE,KAAK,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,qDAAqD;AACrD,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;CACxE;AAED,sEAAsE;AACtE,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAAC;IACpC,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;IACnC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC;CAClE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEhD,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB;IAC9D,kEAAkE;IAClE,QAAQ,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IAClD,+CAA+C;IAC/C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,iCAAiC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE;QACd,QAAQ,CAAC,SAAS,EAAE,OAAO,GAAG,eAAe,GAAG,eAAe,CAAC;QAChE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC3B,CAAC;IACF,kBAAkB;IAClB,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IACvB,yBAAyB;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAClB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAC9B,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAC;KAC3C,CAAC;IACF,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAClB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;QACjC;;;;;WAKG;QACH,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;QACtC;;;;;WAKG;QACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;KACnC,CAAC;IACF,sDAAsD;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IACxC,yDAAyD;IACzD,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;CACzC;AAED,MAAM,MAAM,QAAQ,GAChB;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAAE,GAC5D;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE;AACxD;;;;;;GAMG;GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AAC1G;;;;;;GAMG;GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;AAClI;;;;;;;;;;;;;;GAcG;GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,UAAW,SAAQ,WAAW,CAAC,kBAAkB,EAAE,QAAQ,EAAE,YAAY,CAAC;IACrF,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,QAAQ,CAAkC;IAClD;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoC;gBAEvD,QAAQ,GAAE,OAAO,CAAC,kBAAkB,CAAM;IAEtD,SAAS,CAAC,SAAS,IAAI,MAAM;IAC7B,SAAS,CAAC,eAAe,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAGxD,SAAS,CAAC,sBAAsB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA2BxE,SAAS,CAAC,gBAAgB,IAAI,aAAa,CAAC,MAAM,kBAAkB,CAAC;IACrE,SAAS,CAAC,aAAa,IAAI,MAAM;IAKjC;;;;;OAKG;cACa,mBAAmB,IAAI,OAAO,CAAC,iBAAiB,CAAC;cAgBjD,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;cA+D5B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;cAyB/B,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,YAAY,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3E,SAAS,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI;YAqBzB,YAAY;IAmC1B;;;;;;;OAOG;YACW,eAAe;IAqB7B,OAAO,CAAC,UAAU;CAcnB;AA0BD;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,sBAAsB,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,iBAAiB,CAAC;IACjG,QAAQ,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,iBAAiB,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC,IAAI,EAAE;QACT,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,KAAK,CAAC;YACd,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;YAAC,GAAG,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;YAClD,SAAS,CAAC,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAAC,CAAC;SACnE,CAAC,CAAC;KACJ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACtB;AAED,UAAU,oBAAoB;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;QACvB,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,CAAC;KACtD,CAAC;IACF,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,SAAS,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3E,GAAG,CAAC,IAAI,EAAE;QACR,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;KACzD,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClB;;;;;OAKG;IACH,aAAa,CAAC,CAAC,IAAI,EAAE,aAAa,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;KAClD,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpB;AAYD;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,iBAAiB,CAAC,CAAC,EACvC,IAAI,EAAE;IACJ,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B,EACD,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACrB,OAAO,CAAC,CAAC,CAAC,CAeZ"}

package/dist/io/broker/KafkaActor.js

@@ -0,0 +1,355 @@
+import { match } from 'ts-pattern';
+import { ConfigKeys } from '../../config/ConfigKeys.js';
+import { Lazy } from '../../util/Lazy.js';
+import { lazyImportModule } from '../../util/LazyImport.js';
+import { BrokerActor } from './BrokerActor.js';
+/**
+ * Kafka producer + consumer in one actor, backed by `kafkajs`.  When
+ * `consumer.groupId` is set, a consumer is started after `connectImpl`
+ * and consumed records are delivered to `target`.  When a producer is
+ * the only goal, leave `consumer` and `topics` empty.
+ *
+ * **Offset-commit semantics.**
+ *
+ *   - `commitMode: 'auto'` (default) — kafkajs commits after each
+ *     handler returns successfully → **at-least-once**.  Cheap; OK
+ *     for idempotent handlers.
+ *   - `commitMode: 'manual'` — pump pauses on each message and waits
+ *     for an explicit `commit` command from the handler (#2).  The
+ *     handler is responsible for sending exactly one `commit` (or
+ *     `nack`) per delivered record.  If neither arrives within
+ *     `commitTimeoutMs` the pump rejects internally, kafkajs treats
+ *     the partition as failed, and re-delivery happens on rebalance.
+ *     Produces **exactly-once-with-processing**: a message that
+ *     successfully passed through `commit` is committed; a crash or
+ *     `nack` re-delivers.
+ *
+ *   const kafka = system.spawnAnonymous(Props.create(() => new KafkaActor({
+ *     brokers: ['kafka:9092'],
+ *     consumer: { groupId: 'orders', commitMode: 'manual' },
+ *     topics: ['orders'],
+ *     target: orderProcessor,
+ *   })));
+ *
+ *   class OrderProcessor extends Actor<KafkaRecord> {
+ *     constructor(private readonly kafka: ActorRef<KafkaCmd>) { super(); }
+ *     async onReceive(rec: KafkaRecord) {
+ *       try {
+ *         await db.insertOrder(JSON.parse(rec.value!.toString()));
+ *         this.kafka.tell({ kind: 'commit', topic: rec.topic,
+ *                            partition: rec.partition, offset: rec.offset });
+ *       } catch (e) {
+ *         this.kafka.tell({ kind: 'nack',   topic: rec.topic,
+ *                            partition: rec.partition, offset: rec.offset });
+ *       }
+ *     }
+ *   }
+ */
+export class KafkaActor extends BrokerActor {
+    kafka = null;
+    producer = null;
+    consumer = null;
+    /**
+     * Map of `<topic>|<partition>|<offset>` → in-flight commit promise
+     * resolver.  Only populated in `commitMode: 'manual'`; entries are
+     * inserted by the eachMessage pump and removed by `commit` /
+     * `nack` / the timeout.
+     */
+    pendingCommits = new Map();
+    constructor(settings = {}) { super(settings); }
+    configKey() { return ConfigKeys.io.broker.kafka; }
+    builtInDefaults() {
+        return { ssl: false, producer: { idempotent: false, allowAutoTopicCreation: false } };
+    }
+    readSettingsFromConfig(c) {
+        const out = {};
+        if (c.hasPath('brokers'))
+            out.brokers = c.getStringList('brokers');
+        if (c.hasPath('clientId'))
+            out.clientId = c.getString('clientId');
+        if (c.hasPath('ssl'))
+            out.ssl = c.getBoolean('ssl');
+        if (c.hasPath('sasl')) {
+            const s = c.getConfig('sasl');
+            out.sasl = {
+                mechanism: s.getString('mechanism'),
+                username: s.getString('username'),
+                password: s.getString('password'),
+            };
+        }
+        if (c.hasPath('consumer')) {
+            const cc = c.getConfig('consumer');
+            out.consumer = {
+                groupId: cc.hasPath('groupId') ? cc.getString('groupId') : undefined,
+                fromBeginning: cc.hasPath('fromBeginning') ? cc.getBoolean('fromBeginning') : undefined,
+                commitMode: cc.hasPath('commitMode')
+                    ? cc.getString('commitMode')
+                    : undefined,
+                commitTimeoutMs: cc.hasPath('commitTimeoutMs') ? cc.getNumber('commitTimeoutMs') : undefined,
+            };
+        }
+        if (c.hasPath('topics'))
+            out.topics = c.getStringList('topics');
+        return out;
+    }
+    requiredSettings() { return ['brokers']; }
+    endpointLabel() {
+        const brokers = this.settings.brokers;
+        return Array.isArray(brokers) ? `kafka://${brokers.join(',')}` : `kafka://${brokers ?? ''}`;
+    }
+    /**
+     * Build a `KafkaInstanceLike` from the configured settings.  Override
+     * in a subclass for tests that want to inject mock producers /
+     * consumers without going through the kafkajs peer dep — that's the
+     * test seam used by `tests/unit/io/broker/KafkaActor.test.ts`.
+     */
+    async createKafkaInstance() {
+        const kafkajs = await kafkaLazy.get();
+        const Ctor = kafkajs.Kafka ?? kafkajs.default.Kafka;
+        const brokersRaw = this.settings.brokers;
+        const brokers = Array.isArray(brokersRaw)
+            ? brokersRaw
+            : (typeof brokersRaw === 'string' ? brokersRaw : '')
+                .split(',').map((s) => s.trim()).filter(Boolean);
+        return new Ctor({
+            clientId: this.settings.clientId,
+            brokers: [...brokers],
+            ssl: this.settings.ssl,
+            sasl: this.settings.sasl,
+        });
+    }
+    async connectImpl() {
+        this.kafka = await this.createKafkaInstance();
+        this.producer = this.kafka.producer({
+            idempotent: this.settings.producer?.idempotent,
+            allowAutoTopicCreation: this.settings.producer?.allowAutoTopicCreation,
+        });
+        await this.producer.connect();
+        if (this.settings.consumer?.groupId) {
+            this.consumer = this.kafka.consumer({ groupId: this.settings.consumer.groupId });
+            await this.consumer.connect();
+            for (const topic of this.settings.topics ?? []) {
+                await this.consumer.subscribe({
+                    topic, fromBeginning: this.settings.consumer.fromBeginning ?? false,
+                });
+            }
+            const target = this.settings.target;
+            const manualCommit = this.settings.consumer.commitMode === 'manual';
+            const commitTimeoutMs = this.settings.consumer.commitTimeoutMs ?? 30_000;
+            // We deliberately don't await `run` — it's a long-running pump.
+            void this.consumer.run({
+                // kafkajs v2: `autoCommit: false` disables the auto-commit
+                // path so eachMessage's resolution doesn't trigger a commit.
+                // We drive commits from the actor via `commitOffsets` instead.
+                autoCommit: !manualCommit,
+                eachMessage: async ({ topic, partition, message, heartbeat }) => {
+                    if (!target)
+                        return;
+                    target.tell({
+                        topic, partition,
+                        offset: message.offset,
+                        key: message.key,
+                        value: message.value,
+                        timestamp: message.timestamp,
+                        headers: message.headers ?? {},
+                    });
+                    if (!manualCommit)
+                        return;
+                    // Wait for an external `commit` / `nack` before resolving.
+                    // The promise rejects on `nack`, the timeout, or disconnect
+                    // cleanup — kafkajs treats a rejected eachMessage as a
+                    // partition failure → rebalance → re-delivery (#2).
+                    const key = pendingKey(topic, partition, message.offset);
+                    await new Promise((resolve, reject) => {
+                        const timer = setTimeout(() => {
+                            this.pendingCommits.delete(key);
+                            reject(new Error(`KafkaActor: no commit/nack received for ${key} within ${commitTimeoutMs}ms`));
+                        }, commitTimeoutMs);
+                        this.pendingCommits.set(key, {
+                            done: () => { clearTimeout(timer); resolve(); },
+                            fail: (err) => { clearTimeout(timer); reject(err); },
+                            heartbeat,
+                            topic, partition, offset: message.offset,
+                        });
+                    });
+                },
+            });
+        }
+    }
+    async disconnectImpl() {
+        // Reject every still-pending commit so the kafkajs eachMessage
+        // pump unwinds cleanly — otherwise consumer.disconnect would
+        // hang waiting for the in-flight handler to resolve.
+        if (this.pendingCommits.size > 0) {
+            for (const p of this.pendingCommits.values()) {
+                p.fail(new Error('KafkaActor: disconnecting before commit/nack arrived'));
+            }
+            this.pendingCommits.clear();
+        }
+        const errors = [];
+        if (this.consumer) {
+            try {
+                await this.consumer.disconnect();
+            }
+            catch (e) {
+                errors.push(e);
+            }
+            this.consumer = null;
+        }
+        if (this.producer) {
+            try {
+                await this.producer.disconnect();
+            }
+            catch (e) {
+                errors.push(e);
+            }
+            this.producer = null;
+        }
+        this.kafka = null;
+        if (errors.length > 0) {
+            this.log.warn(`KafkaActor disconnect: ${errors.map((e) => e.message).join('; ')}`);
+        }
+    }
+    async dispatchOutgoing(env) {
+        if (!this.producer)
+            throw new Error('KafkaActor: producer not connected');
+        const p = env.payload;
+        const value = typeof p.value === 'string' ? Buffer.from(p.value) : p.value;
+        const key = p.key === undefined ? null
+            : (typeof p.key === 'string' ? Buffer.from(p.key) : p.key);
+        await this.producer.send({
+            topic: p.topic,
+            messages: [{ value, key, partition: p.partition, headers: p.headers }],
+        });
+    }
+    onReceive(cmd) {
+        // Compile-time exhaustiveness: adding a new KafkaCmd variant
+        // forces this site to handle it explicitly.
+        match(cmd)
+            .with({ kind: 'publish' }, (c) => {
+            this.enqueueOutbound(c.publish);
+        })
+            .with({ kind: 'subscribe' }, (c) => {
+            // Runtime topic-add — kafkajs requires the consumer already be running.
+            if (this.consumer && this.connectionState === 'connected') {
+                void this.consumer.subscribe({ topic: c.topic, fromBeginning: false });
+            }
+        })
+            .with({ kind: 'commit' }, (c) => { void this.handleCommit(c); })
+            .with({ kind: 'nack' }, (c) => this.handleNack(c))
+            .with({ kind: 'heartbeat' }, (c) => { void this.handleHeartbeat(c); })
+            .exhaustive();
+    }
+    /* ----------------------------- internals ------------------------------ */
+    async handleCommit(cmd) {
+        const key = pendingKey(cmd.topic, cmd.partition, cmd.offset);
+        const pending = this.pendingCommits.get(key);
+        if (!pending) {
+            // Already committed, expired, or — most commonly — auto-mode
+            // (where no pending entry was ever inserted).  Silent no-op
+            // either way: handlers can be written to send `commit`
+            // unconditionally without checking the configured mode.
+            this.log.debug(`KafkaActor: commit for unknown ${key} — ignored (already committed, expired, or auto-mode)`);
+            return;
+        }
+        if (!this.consumer || !this.consumer.commitOffsets) {
+            pending.fail(new Error('KafkaActor: commit arrived but consumer is not connected or kafkajs lacks commitOffsets'));
+            this.pendingCommits.delete(key);
+            return;
+        }
+        try {
+            // kafkajs `commitOffsets` takes the **next** offset to consume,
+            // i.e., one past the message we just processed.  Offsets are
+            // decimal strings; we use BigInt so very large values stay
+            // exact (Kafka offsets are 64-bit).
+            const next = String(BigInt(cmd.offset) + 1n);
+            await this.consumer.commitOffsets([
+                { topic: cmd.topic, partition: cmd.partition, offset: next },
+            ]);
+            pending.done();
+        }
+        catch (err) {
+            pending.fail(err instanceof Error ? err : new Error(String(err)));
+        }
+        finally {
+            this.pendingCommits.delete(key);
+        }
+    }
+    /**
+     * Forward a `heartbeat` command to the captured kafkajs callback —
+     * keeps the record's session-deadline alive without affecting the
+     * offset.  Unknown record (auto-mode, already-committed, expired)
+     * is a silent no-op so handlers can call this unconditionally
+     * without checking `commitMode` or whether the record's commit has
+     * already landed elsewhere.
+     */
+    async handleHeartbeat(cmd) {
+        const key = pendingKey(cmd.topic, cmd.partition, cmd.offset);
+        const pending = this.pendingCommits.get(key);
+        if (!pending || !pending.heartbeat) {
+            this.log.debug(`KafkaActor: heartbeat for unknown ${key} — ignored`);
+            return;
+        }
+        try {
+            await pending.heartbeat();
+        }
+        catch (err) {
+            // A failing heartbeat shouldn't reject the in-flight commit —
+            // the handler still has a chance to commit/nack normally.  We
+            // just log and let the existing timeout / handler path run.
+            this.log.warn(`KafkaActor: heartbeat call failed for ${key}: ${err.message}`);
+        }
+    }
+    handleNack(cmd) {
+        const key = pendingKey(cmd.topic, cmd.partition, cmd.offset);
+        const pending = this.pendingCommits.get(key);
+        if (!pending)
+            return;
+        this.log.warn(`KafkaActor: nack for ${key}${cmd.reason ? ` (${cmd.reason})` : ''} — `
+            + `re-delivery will happen on next rebalance`);
+        pending.fail(new Error(cmd.reason ?? 'KafkaActor: nack from handler'));
+        this.pendingCommits.delete(key);
+    }
+}
+function pendingKey(topic, partition, offset) {
+    return `${topic}|${partition}|${offset}`;
+}
+const kafkaLazy = Lazy.of(() => lazyImportModule('kafkajs', { context: 'KafkaActor' }));
+/**
+ * Run `body` while periodically telling `kafka` to heartbeat the
+ * record.  Cancels the timer on success or failure; the body's
+ * resolution / rejection is propagated.  Use this when a manual-commit
+ * handler is likely to exceed the consumer's `sessionTimeoutMs / 3` —
+ * the rule of thumb being three heartbeats per session window so a
+ * single dropped tick doesn't trigger eviction.
+ *
+ * Default `everyMs` is 10 s, suitable for kafkajs's default 30 s
+ * `sessionTimeout`.  If you tuned the consumer's session timeout up,
+ * scale `everyMs` to match.
+ *
+ * @example
+ * ```ts
+ * await withAutoHeartbeat({ kafka, record: rec, everyMs: 5_000 }, async () => {
+ *   await reallySlowDatabaseWork(rec.value);
+ * });
+ * kafka.tell({ kind: 'commit', ...rec });
+ * ```
+ */
+export async function withAutoHeartbeat(args, body) {
+    const intervalMs = args.everyMs ?? 10_000;
+    const timer = setInterval(() => {
+        args.kafka.tell({
+            kind: 'heartbeat',
+            topic: args.record.topic,
+            partition: args.record.partition,
+            offset: args.record.offset,
+        });
+    }, intervalMs);
+    try {
+        return await body();
+    }
+    finally {
+        clearInterval(timer);
+    }
+}
+//# sourceMappingURL=KafkaActor.js.map