@flowcore/data-pump 0.16.5 → 0.17.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [0.17.0](https://github.com/flowcore-io/data-pump/compare/v0.16.5...v0.17.0) (2026-03-19)
+
+
+### Features
+
+* add cluster mode with leader election and WebSocket worker mesh ([34ecce7](https://github.com/flowcore-io/data-pump/commit/34ecce7d80413364e96a21c634f103b1da1576c6))
+* cluster mode with leader election & WebSocket worker mesh ([d2e2e47](https://github.com/flowcore-io/data-pump/commit/d2e2e473770f52c59981dcdf4327f1bb96555784))
+
+
+### Bug Fixes
+
+* correct FlowcoreEvent type in test fixture ([3139702](https://github.com/flowcore-io/data-pump/commit/31397024faa7cd259d9c63334f1258b6eceaac96))
+* resolve deno lint issues (require-await, unused imports) ([d5ef0ba](https://github.com/flowcore-io/data-pump/commit/d5ef0bafc6956793579f30e609d09adefb2b6128))
+
 ## [0.16.5](https://github.com/flowcore-io/data-pump/compare/v0.16.4...v0.16.5) (2026-03-18)
 
 

package/esm/data-pump/data-pump-cluster.d.ts

@@ -0,0 +1,80 @@
+import * as dntShim from "../_dnt.shims.js";
+import { type FlowcoreDataPumpOptions } from "./data-pump.js";
+import type { FlowcoreDataPumpCoordinator } from "./types.js";
+export interface FlowcoreDataPumpClusterOptions extends FlowcoreDataPumpOptions {
+    coordinator: FlowcoreDataPumpCoordinator;
+    advertisedAddress: string;
+    leaseTtlMs?: number;
+    leaseRenewIntervalMs?: number;
+    heartbeatIntervalMs?: number;
+    workerConcurrency?: number;
+}
+export declare class FlowcoreDataPumpCluster {
+    private readonly options;
+    private readonly instanceId;
+    private readonly coordinator;
+    private readonly leaseTtlMs;
+    private readonly leaseRenewIntervalMs;
+    private readonly heartbeatIntervalMs;
+    private readonly workerConcurrency;
+    private readonly logger?;
+    private running;
+    private isLeader;
+    private pump?;
+    private leaderConnection?;
+    private workers;
+    private workerRoundRobinIndex;
+    private leaseRenewInterval?;
+    private discoveryInterval?;
+    private heartbeatInterval?;
+    private electionInterval?;
+    private workerHandler?;
+    private workerFailedHandler?;
+    constructor(options: FlowcoreDataPumpClusterOptions);
+    get isRunning(): boolean;
+    get isLeaderInstance(): boolean;
+    get activeWorkerCount(): number;
+    get id(): string;
+    /**
+     * Handle an incoming WebSocket connection from a peer.
+     * Call this from your HTTP server's WebSocket upgrade handler.
+     *
+     * Example (Deno):
+     * ```typescript
+     * Deno.serve({ port: 8080 }, (req) => {
+     *   if (req.headers.get("upgrade") === "websocket") {
+     *     const { socket, response } = Deno.upgradeWebSocket(req)
+     *     cluster.handleConnection(socket)
+     *     return response
+     *   }
+     *   return new Response("Not found", { status: 404 })
+     * })
+     * ```
+     *
+     * Example (Node.js with ws):
+     * ```typescript
+     * import { WebSocketServer } from "ws"
+     * const wss = new WebSocketServer({ port: 8080 })
+     * wss.on("connection", (ws) => cluster.handleConnection(ws))
+     * ```
+     */
+    handleConnection(ws: dntShim.WebSocket): void;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    private startHeartbeat;
+    private startElectionLoop;
+    private attemptElection;
+    private becomeLeader;
+    private stepDownAsLeader;
+    private startPumpAsLeader;
+    private distributeEvents;
+    private selectWorker;
+    private startWorkerDiscovery;
+    private discoverWorkers;
+    private connectToWorker;
+    private handleWorkerDisconnect;
+    private scheduleReconnect;
+    private handleLeaderMessage;
+    private handleWorkerMessage;
+}
+//# sourceMappingURL=data-pump-cluster.d.ts.map

package/esm/data-pump/data-pump-cluster.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-pump-cluster.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump-cluster.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAoB,KAAK,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AAE/E,OAAO,KAAK,EAAE,2BAA2B,EAAkB,MAAM,YAAY,CAAA;AAc7E,MAAM,WAAW,8BAA+B,SAAQ,uBAAuB;IAC7E,WAAW,EAAE,2BAA2B,CAAA;IACxC,iBAAiB,EAAE,MAAM,CAAA;IACzB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC3B;AAUD,qBAAa,uBAAuB;IA4BtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IA3BpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;IACzD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAC5C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB;IAExC,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,IAAI,CAAC,CAAkB;IAC/B,OAAO,CAAC,gBAAgB,CAAC,CAAc;IAGvC,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,qBAAqB,CAAI;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAgC;IAC3D,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAG1D,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAC1D,OAAO,CAAC,gBAAgB,CAAC,CAAgC;IAGzD,OAAO,CAAC,aAAa,CAAC,CAA4C;IAClE,OAAO,CAAC,mBAAmB,CAAC,CAAmD;gBAElD,OAAO,EAAE,8BAA8B;IAcpE,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,gBAAgB,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,GAAG,IAAI;IAevC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA2C3B,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,iBAAiB;YAyBX,eAAe;YAgBf,YAAY;IAkB1B,OAAO,CAAC,gBAAgB;IAgCxB,OAAO,CAAC,iBAAiB;YAmBX,gBAAgB;IAqC9B,OAAO,CAAC,YAAY;IAoBpB,OAAO,CAAC,oBAAoB;YAOd,eAAe;IA8B7B,OAAO,CAAC,eAAe;IAqCvB,OAAO,CAAC,sBAAsB;IAiB9B,OAAO,CAAC,iBAAiB;IAiCzB,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,mBAAmB;CA6B5B"}

package/esm/data-pump/data-pump-cluster.js

@@ -0,0 +1,558 @@
+import * as dntShim from "../_dnt.shims.js";
+import { FlowcoreDataPump } from "./data-pump.js";
+import { clusterMetrics } from "./metrics.js";
+import { DeliveryTracker, WsConnection } from "./ws-protocol.js";
+const LEASE_KEY = "flowcore-data-pump-leader";
+const DISCOVERY_POLL_INTERVAL_MS = 10_000;
+const DEFAULT_LEASE_TTL_MS = 30_000;
+const DEFAULT_LEASE_RENEW_INTERVAL_MS = 10_000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 5_000;
+const DEFAULT_STALE_THRESHOLD_MS = 30_000;
+const DEFAULT_WORKER_CONCURRENCY = 1;
+const RECONNECT_BASE_MS = 1_000;
+const RECONNECT_MAX_MS = 30_000;
+const DELIVERY_TIMEOUT_MS = 30_000;
+export class FlowcoreDataPumpCluster {
+    constructor(options) {
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+        Object.defineProperty(this, "instanceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "coordinator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaseTtlMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaseRenewIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "heartbeatIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workerConcurrency", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "running", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "isLeader", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "pump", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaderConnection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // leader state
+        Object.defineProperty(this, "workers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "workerRoundRobinIndex", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "leaseRenewInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "discoveryInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // common state
+        Object.defineProperty(this, "heartbeatInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "electionInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // worker state - handler for processing events received from leader
+        Object.defineProperty(this, "workerHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workerFailedHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.instanceId = crypto.randomUUID();
+        this.coordinator = options.coordinator;
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
+        this.leaseRenewIntervalMs = options.leaseRenewIntervalMs ?? DEFAULT_LEASE_RENEW_INTERVAL_MS;
+        this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
+        this.workerConcurrency = options.workerConcurrency ?? DEFAULT_WORKER_CONCURRENCY;
+        this.logger = options.logger;
+        // capture user's processor for worker mode
+        this.workerHandler = options.processor?.handler;
+        this.workerFailedHandler = options.processor?.failedHandler;
+    }
+    get isRunning() {
+        return this.running;
+    }
+    get isLeaderInstance() {
+        return this.isLeader;
+    }
+    get activeWorkerCount() {
+        return this.workers.size;
+    }
+    get id() {
+        return this.instanceId;
+    }
+    /**
+     * Handle an incoming WebSocket connection from a peer.
+     * Call this from your HTTP server's WebSocket upgrade handler.
+     *
+     * Example (Deno):
+     * ```typescript
+     * Deno.serve({ port: 8080 }, (req) => {
+     *   if (req.headers.get("upgrade") === "websocket") {
+     *     const { socket, response } = Deno.upgradeWebSocket(req)
+     *     cluster.handleConnection(socket)
+     *     return response
+     *   }
+     *   return new Response("Not found", { status: 404 })
+     * })
+     * ```
+     *
+     * Example (Node.js with ws):
+     * ```typescript
+     * import { WebSocketServer } from "ws"
+     * const wss = new WebSocketServer({ port: 8080 })
+     * wss.on("connection", (ws) => cluster.handleConnection(ws))
+     * ```
+     */
+    handleConnection(ws) {
+        this.logger?.debug("Incoming WS connection");
+        const conn = new WsConnection(ws, {
+            logger: this.logger,
+            onMessage: (msg) => this.handleWorkerMessage(msg, conn),
+            onClose: () => {
+                this.logger?.info("Leader connection closed");
+                this.leaderConnection = undefined;
+            },
+        });
+        this.leaderConnection = conn;
+    }
+    async start() {
+        if (this.running) {
+            throw new Error("Cluster already running");
+        }
+        this.running = true;
+        this.logger?.info("Starting cluster instance", { instanceId: this.instanceId });
+        // Step 1: Register in DB
+        await this.coordinator.register(this.instanceId, this.options.advertisedAddress);
+        // Step 2: Start heartbeat
+        this.startHeartbeat();
+        // Step 3: Start leader election loop
+        this.startElectionLoop();
+    }
+    async stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.logger?.info("Stopping cluster instance", { instanceId: this.instanceId });
+        // stop election + heartbeat
+        if (this.electionInterval) {
+            clearInterval(this.electionInterval);
+            this.electionInterval = undefined;
+        }
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = undefined;
+        }
+        // stop leader duties
+        this.stepDownAsLeader();
+        // close worker connection to leader
+        this.leaderConnection?.close();
+        this.leaderConnection = undefined;
+        // unregister
+        try {
+            await this.coordinator.releaseLease(this.instanceId, LEASE_KEY);
+        }
+        catch {
+            // ignore
+        }
+        try {
+            await this.coordinator.unregister(this.instanceId);
+        }
+        catch {
+            // ignore
+        }
+        clusterMetrics.leaderStatusGauge.set(0);
+        clusterMetrics.activeWorkersGauge.set(0);
+        this.logger?.info("Cluster instance stopped");
+    }
+    // #region Heartbeat
+    startHeartbeat() {
+        this.heartbeatInterval = setInterval(async () => {
+            if (!this.running)
+                return;
+            try {
+                await this.coordinator.heartbeat(this.instanceId);
+            }
+            catch (error) {
+                this.logger?.warn("Heartbeat failed", { error });
+            }
+        }, this.heartbeatIntervalMs);
+    }
+    // #endregion
+    // #region Leader Election
+    startElectionLoop() {
+        // attempt immediately
+        void this.attemptElection();
+        this.electionInterval = setInterval(async () => {
+            if (!this.running)
+                return;
+            if (this.isLeader) {
+                // renew lease
+                try {
+                    const renewed = await this.coordinator.renewLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+                    if (!renewed) {
+                        this.logger?.warn("Lease renewal failed, stepping down");
+                        this.stepDownAsLeader();
+                    }
+                }
+                catch (error) {
+                    this.logger?.error("Lease renewal error, stepping down", { error });
+                    this.stepDownAsLeader();
+                }
+            }
+            else {
+                await this.attemptElection();
+            }
+        }, this.leaseRenewIntervalMs);
+    }
+    async attemptElection() {
+        try {
+            const acquired = await this.coordinator.acquireLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+            if (acquired && !this.isLeader) {
+                this.logger?.info("Won leader election", { instanceId: this.instanceId });
+                await this.becomeLeader();
+            }
+        }
+        catch (error) {
+            this.logger?.debug("Election attempt failed", { error });
+        }
+    }
+    // #endregion
+    // #region Leader Mode
+    async becomeLeader() {
+        this.isLeader = true;
+        clusterMetrics.leaderStatusGauge.set(1);
+        // close any incoming leader connection (we are leader now)
+        this.leaderConnection?.close();
+        this.leaderConnection = undefined;
+        // start worker discovery
+        this.startWorkerDiscovery();
+        // initial discovery
+        await this.discoverWorkers();
+        // start the pump with distribution processor
+        this.startPumpAsLeader();
+    }
+    stepDownAsLeader() {
+        if (!this.isLeader)
+            return;
+        this.isLeader = false;
+        clusterMetrics.leaderStatusGauge.set(0);
+        // stop discovery
+        if (this.discoveryInterval) {
+            clearInterval(this.discoveryInterval);
+            this.discoveryInterval = undefined;
+        }
+        // stop lease renewal (separate from election interval)
+        if (this.leaseRenewInterval) {
+            clearInterval(this.leaseRenewInterval);
+            this.leaseRenewInterval = undefined;
+        }
+        // disconnect all workers
+        for (const worker of this.workers.values()) {
+            worker.deliveryTracker.rejectAll(new Error("Leader stepping down"));
+            worker.connection.close();
+        }
+        this.workers.clear();
+        clusterMetrics.activeWorkersGauge.set(0);
+        // stop pump
+        if (this.pump) {
+            this.pump.stop();
+            this.pump = undefined;
+        }
+    }
+    startPumpAsLeader() {
+        // create pump with distribution processor that sends events to workers
+        const pumpOptions = {
+            ...this.options,
+            processor: {
+                concurrency: this.workerConcurrency,
+                handler: async (events) => {
+                    await this.distributeEvents(events);
+                },
+                failedHandler: this.workerFailedHandler,
+            },
+        };
+        this.pump = FlowcoreDataPump.create(pumpOptions);
+        this.pump.start().catch((error) => {
+            this.logger?.error("Pump error in leader mode", { error });
+        });
+    }
+    async distributeEvents(events) {
+        const worker = this.selectWorker();
+        if (!worker) {
+            // no workers available - process locally (single-instance fallback)
+            this.logger?.debug("No workers available, processing locally");
+            if (this.workerHandler) {
+                await this.workerHandler(events);
+            }
+            return;
+        }
+        const deliveryId = crypto.randomUUID();
+        const eventIds = events.map((e) => e.eventId);
+        worker.pendingCount++;
+        clusterMetrics.eventsDistributedCounter.inc(events.length);
+        // send events to worker
+        worker.connection.send({
+            type: "events",
+            deliveryId,
+            events,
+        });
+        // wait for ack
+        try {
+            await worker.deliveryTracker.add(deliveryId, eventIds, DELIVERY_TIMEOUT_MS);
+            worker.pendingCount--;
+            clusterMetrics.workerAcksCounter.inc();
+        }
+        catch (error) {
+            worker.pendingCount--;
+            clusterMetrics.workerFailsCounter.inc();
+            throw error;
+        }
+    }
+    selectWorker() {
+        const available = [];
+        for (const worker of this.workers.values()) {
+            if (worker.pendingCount < this.workerConcurrency && !worker.connection.isClosed) {
+                available.push(worker);
+            }
+        }
+        if (available.length === 0)
+            return null;
+        this.workerRoundRobinIndex = this.workerRoundRobinIndex % available.length;
+        const selected = available[this.workerRoundRobinIndex];
+        this.workerRoundRobinIndex++;
+        return selected;
+    }
+    // #endregion
+    // #region Worker Discovery
+    startWorkerDiscovery() {
+        this.discoveryInterval = setInterval(async () => {
+            if (!this.running || !this.isLeader)
+                return;
+            await this.discoverWorkers();
+        }, DISCOVERY_POLL_INTERVAL_MS);
+    }
+    async discoverWorkers() {
+        try {
+            const instances = await this.coordinator.getInstances(DEFAULT_STALE_THRESHOLD_MS);
+            for (const instance of instances) {
+                // skip self
+                if (instance.instanceId === this.instanceId)
+                    continue;
+                // skip already connected
+                if (this.workers.has(instance.instanceId))
+                    continue;
+                this.connectToWorker(instance.instanceId, instance.address);
+            }
+            // clean up workers that are no longer registered
+            const registeredIds = new Set(instances.map((i) => i.instanceId));
+            for (const [id, worker] of this.workers) {
+                if (!registeredIds.has(id)) {
+                    this.logger?.info("Worker no longer registered, disconnecting", { instanceId: id });
+                    worker.deliveryTracker.rejectAll(new Error("Worker unregistered"));
+                    worker.connection.close();
+                    this.workers.delete(id);
+                }
+            }
+            clusterMetrics.activeWorkersGauge.set(this.workers.size);
+        }
+        catch (error) {
+            this.logger?.warn("Worker discovery failed", { error });
+        }
+    }
+    connectToWorker(instanceId, address) {
+        this.logger?.info("Connecting to worker", { instanceId, address });
+        try {
+            const ws = new dntShim.WebSocket(address);
+            const deliveryTracker = new DeliveryTracker();
+            const workerConn = {
+                instanceId,
+                connection: undefined,
+                deliveryTracker,
+                pendingCount: 0,
+                reconnectAttempts: 0,
+            };
+            ws.onopen = () => {
+                const conn = new WsConnection(ws, {
+                    logger: this.logger,
+                    onMessage: (msg) => this.handleLeaderMessage(msg, instanceId),
+                    onClose: () => this.handleWorkerDisconnect(instanceId),
+                });
+                workerConn.connection = conn;
+                workerConn.reconnectAttempts = 0;
+                this.workers.set(instanceId, workerConn);
+                clusterMetrics.activeWorkersGauge.set(this.workers.size);
+                this.logger?.info("Connected to worker", { instanceId });
+            };
+            ws.onerror = () => {
+                this.logger?.warn("Failed to connect to worker", { instanceId, address });
+                this.scheduleReconnect(instanceId, address, (workerConn.reconnectAttempts || 0) + 1);
+            };
+        }
+        catch (error) {
+            this.logger?.warn("Error connecting to worker", { instanceId, error });
+        }
+    }
+    handleWorkerDisconnect(instanceId) {
+        const worker = this.workers.get(instanceId);
+        if (!worker)
+            return;
+        this.logger?.info("Worker disconnected", { instanceId });
+        // reject all pending deliveries - pump's ack timeout will handle reOpen
+        worker.deliveryTracker.rejectAll(new Error("Worker disconnected"));
+        this.workers.delete(instanceId);
+        clusterMetrics.activeWorkersGauge.set(this.workers.size);
+        // schedule reconnect if still running as leader
+        if (this.running && this.isLeader) {
+            this.scheduleReconnect(instanceId, "", worker.reconnectAttempts + 1);
+        }
+    }
+    scheduleReconnect(instanceId, address, attempts) {
+        if (!this.running || !this.isLeader)
+            return;
+        const delay = Math.min(RECONNECT_BASE_MS * Math.pow(2, attempts - 1), RECONNECT_MAX_MS);
+        this.logger?.debug("Scheduling reconnect", { instanceId, delay, attempts });
+        setTimeout(async () => {
+            if (!this.running || !this.isLeader)
+                return;
+            // re-discover address from DB if we don't have it
+            let resolvedAddress = address;
+            if (!resolvedAddress) {
+                try {
+                    const instances = await this.coordinator.getInstances(DEFAULT_STALE_THRESHOLD_MS);
+                    const instance = instances.find((i) => i.instanceId === instanceId);
+                    if (!instance)
+                        return; // worker gone, don't reconnect
+                    resolvedAddress = instance.address;
+                }
+                catch {
+                    return;
+                }
+            }
+            if (!this.workers.has(instanceId)) {
+                this.connectToWorker(instanceId, resolvedAddress);
+            }
+        }, delay);
+    }
+    // #endregion
+    // #region Message Handlers
+    // leader receives ack/fail from workers
+    handleLeaderMessage(msg, workerInstanceId) {
+        const worker = this.workers.get(workerInstanceId);
+        if (!worker)
+            return;
+        if (msg.type === "ack") {
+            worker.deliveryTracker.ack(msg.deliveryId);
+        }
+        else if (msg.type === "fail") {
+            worker.deliveryTracker.fail(msg.deliveryId);
+        }
+    }
+    // worker receives events from leader
+    handleWorkerMessage(msg, conn) {
+        if (msg.type !== "events")
+            return;
+        const { deliveryId, events } = msg;
+        this.logger?.debug("Received events from leader", { deliveryId, count: events.length });
+        // process events using user's handler
+        void (async () => {
+            try {
+                if (this.workerHandler) {
+                    await this.workerHandler(events);
+                }
+                conn.send({
+                    type: "ack",
+                    deliveryId,
+                    eventIds: events.map((e) => e.eventId),
+                });
+            }
+            catch (error) {
+                this.logger?.error("Worker failed to process events", { deliveryId, error });
+                conn.send({
+                    type: "fail",
+                    deliveryId,
+                    eventIds: events.map((e) => e.eventId),
+                });
+            }
+        })();
+    }
+}

package/esm/data-pump/data-pump.d.ts

@@ -1,4 +1,4 @@
-import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
 import { FlowcoreDataSource } from "./data-source.js";
 import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpProcessor, FlowcoreDataPumpState, FlowcoreDataPumpStateManager, FlowcoreLogger } from "./types.js";
 interface FlowcoreDataPumpNotifierNatsOptions {

package/esm/data-pump/data-source.d.ts

@@ -2,7 +2,7 @@
  * @module data-source
  * @description Provides functionality to interact with Flowcore data sources
  */
-import { type EventListOutput, type FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import { type EventListOutput, type FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
 import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpState } from "./types.js";
 /**
  * Options for configuring a Flowcore data source

package/esm/data-pump/data-source.js

@@ -2,7 +2,7 @@
  * @module data-source
  * @description Provides functionality to interact with Flowcore data sources
  */
-import { DataCoreFetchCommand, EventListCommand, EventTypeListCommand, FlowTypeFetchCommand, TenantTranslateNameToIdCommand, TimeBucketListCommand, } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import { DataCoreFetchCommand, EventListCommand, EventTypeListCommand, FlowTypeFetchCommand, TenantTranslateNameToIdCommand, TimeBucketListCommand, } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
 import { getFlowcoreClient } from "./flowcore-client.js";
 /**
  * Provides methods to interact with a Flowcore data source