@flowcore/pathways 0.19.0 → 0.20.0

package/esm/deps/jsr.io/@flowcore/data-pump/0.18.0/src/data-pump/data-pump-cluster.js

@@ -0,0 +1,654 @@
+import * as dntShim from "../../../../../../../_dnt.shims.js";
+import { FlowcoreDataPump } from "./data-pump.js";
+import { clusterMetrics } from "./metrics.js";
+import { NatsConnectionManager } from "./nats-connection.js";
+import { NatsDistributionLeader, NatsDistributionWorker } from "./nats-distribution.js";
+import { DeliveryTracker, WsConnection } from "./ws-protocol.js";
+const LEASE_KEY = "flowcore-data-pump-leader";
+const DISCOVERY_POLL_INTERVAL_MS = 10_000;
+const DEFAULT_LEASE_TTL_MS = 30_000;
+const DEFAULT_LEASE_RENEW_INTERVAL_MS = 10_000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 5_000;
+const DEFAULT_STALE_THRESHOLD_MS = 30_000;
+const DEFAULT_WORKER_CONCURRENCY = 1;
+const RECONNECT_BASE_MS = 1_000;
+const RECONNECT_MAX_MS = 30_000;
+const DELIVERY_TIMEOUT_MS = 30_000;
+export class FlowcoreDataPumpCluster {
+    constructor(options) {
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+        Object.defineProperty(this, "instanceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "coordinator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaseTtlMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaseRenewIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "heartbeatIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workerConcurrency", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "running", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "isLeader", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "pump", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaderConnection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // leader state
+        Object.defineProperty(this, "workers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "workerRoundRobinIndex", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "leaseRenewInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "discoveryInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // common state
+        Object.defineProperty(this, "heartbeatInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "electionInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // worker state - handler for processing events received from leader
+        Object.defineProperty(this, "workerHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workerFailedHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // NATS distribution state
+        Object.defineProperty(this, "natsConnectionManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "natsDistLeader", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "natsDistWorker", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // validate: WS mode requires advertisedAddress
+        if (!this.useNatsDistribution && !options.advertisedAddress) {
+            throw new Error("advertisedAddress is required when not using NATS distribution (notifier.type !== 'nats')");
+        }
+        this.instanceId = crypto.randomUUID();
+        this.coordinator = options.coordinator;
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
+        this.leaseRenewIntervalMs = options.leaseRenewIntervalMs ?? DEFAULT_LEASE_RENEW_INTERVAL_MS;
+        this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
+        this.workerConcurrency = options.workerConcurrency ?? DEFAULT_WORKER_CONCURRENCY;
+        this.logger = options.logger;
+        // capture user's processor for worker mode
+        this.workerHandler = options.processor?.handler;
+        this.workerFailedHandler = options.processor?.failedHandler;
+    }
+    get useNatsDistribution() {
+        return this.options.notifier?.type === "nats";
+    }
+    get natsDistributionSubject() {
+        const key = this.options.clusterKey ??
+            `${this.options.dataSource.tenant}.${this.options.dataSource.dataCore}.${this.options.dataSource.flowType}`;
+        return `data-pump.distribute.${key}`;
+    }
+    get isRunning() {
+        return this.running;
+    }
+    get isLeaderInstance() {
+        return this.isLeader;
+    }
+    get activeWorkerCount() {
+        return this.workers.size;
+    }
+    get id() {
+        return this.instanceId;
+    }
+    /**
+     * Handle an incoming WebSocket connection from a peer.
+     * Call this from your HTTP server's WebSocket upgrade handler.
+     *
+     * Example (Deno):
+     * ```typescript
+     * Deno.serve({ port: 8080 }, (req) => {
+     *   if (req.headers.get("upgrade") === "websocket") {
+     *     const { socket, response } = Deno.upgradeWebSocket(req)
+     *     cluster.handleConnection(socket)
+     *     return response
+     *   }
+     *   return new Response("Not found", { status: 404 })
+     * })
+     * ```
+     *
+     * Example (Node.js with ws):
+     * ```typescript
+     * import { WebSocketServer } from "ws"
+     * const wss = new WebSocketServer({ port: 8080 })
+     * wss.on("connection", (ws) => cluster.handleConnection(ws))
+     * ```
+     */
+    handleConnection(ws) {
+        this.logger?.debug("Incoming WS connection");
+        const conn = new WsConnection(ws, {
+            logger: this.logger,
+            onMessage: (msg) => this.handleWorkerMessage(msg, conn),
+            onClose: () => {
+                this.logger?.info("Leader connection closed");
+                this.leaderConnection = undefined;
+            },
+        });
+        this.leaderConnection = conn;
+    }
+    async start() {
+        if (this.running) {
+            throw new Error("Cluster already running");
+        }
+        this.running = true;
+        this.logger?.info("Starting cluster instance", { instanceId: this.instanceId });
+        if (this.useNatsDistribution) {
+            // NATS mode: connect shared NATS, start worker subscription
+            const natsServers = this.options.notifier?.type === "nats" ? this.options.notifier.servers : [];
+            this.natsConnectionManager = new NatsConnectionManager(natsServers, this.logger);
+            const conn = await this.natsConnectionManager.connect();
+            // all instances subscribe as workers (leader also processes if no workers pick up)
+            if (this.workerHandler) {
+                this.natsDistWorker = new NatsDistributionWorker(conn, this.natsDistributionSubject, this.workerHandler, this.logger);
+                this.natsDistWorker.start();
+            }
+            // register with placeholder address (not used for routing in NATS mode)
+            await this.coordinator.register(this.instanceId, "nats://distributed");
+        }
+        else {
+            // WS mode: register with actual address
+            await this.coordinator.register(this.instanceId, this.options.advertisedAddress);
+        }
+        // Start heartbeat
+        this.startHeartbeat();
+        // Start leader election loop
+        this.startElectionLoop();
+    }
+    async stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.logger?.info("Stopping cluster instance", { instanceId: this.instanceId });
+        // stop election + heartbeat
+        if (this.electionInterval) {
+            clearInterval(this.electionInterval);
+            this.electionInterval = undefined;
+        }
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = undefined;
+        }
+        // stop leader duties
+        this.stepDownAsLeader();
+        // close worker connection to leader
+        this.leaderConnection?.close();
+        this.leaderConnection = undefined;
+        // stop NATS distribution
+        this.natsDistWorker?.stop();
+        this.natsDistWorker = undefined;
+        this.natsDistLeader = undefined;
+        // close shared NATS connection
+        if (this.natsConnectionManager) {
+            await this.natsConnectionManager.close();
+            this.natsConnectionManager = undefined;
+        }
+        // unregister
+        try {
+            await this.coordinator.releaseLease(this.instanceId, LEASE_KEY);
+        }
+        catch {
+            // ignore
+        }
+        try {
+            await this.coordinator.unregister(this.instanceId);
+        }
+        catch {
+            // ignore
+        }
+        clusterMetrics.leaderStatusGauge.set(0);
+        clusterMetrics.activeWorkersGauge.set(0);
+        this.logger?.info("Cluster instance stopped");
+    }
+    // #region Heartbeat
+    startHeartbeat() {
+        this.heartbeatInterval = setInterval(async () => {
+            if (!this.running)
+                return;
+            try {
+                await this.coordinator.heartbeat(this.instanceId);
+            }
+            catch (error) {
+                this.logger?.warn("Heartbeat failed", { error });
+            }
+        }, this.heartbeatIntervalMs);
+    }
+    // #endregion
+    // #region Leader Election
+    startElectionLoop() {
+        // attempt immediately
+        void this.attemptElection();
+        this.electionInterval = setInterval(async () => {
+            if (!this.running)
+                return;
+            if (this.isLeader) {
+                // renew lease
+                try {
+                    const renewed = await this.coordinator.renewLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+                    if (!renewed) {
+                        this.logger?.warn("Lease renewal failed, stepping down");
+                        this.stepDownAsLeader();
+                    }
+                }
+                catch (error) {
+                    this.logger?.error("Lease renewal error, stepping down", { error });
+                    this.stepDownAsLeader();
+                }
+            }
+            else {
+                await this.attemptElection();
+            }
+        }, this.leaseRenewIntervalMs);
+    }
+    async attemptElection() {
+        try {
+            const acquired = await this.coordinator.acquireLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+            if (acquired && !this.isLeader) {
+                this.logger?.info("Won leader election", { instanceId: this.instanceId });
+                await this.becomeLeader();
+            }
+        }
+        catch (error) {
+            this.logger?.debug("Election attempt failed", { error });
+        }
+    }
+    // #endregion
+    // #region Leader Mode
+    async becomeLeader() {
+        this.isLeader = true;
+        clusterMetrics.leaderStatusGauge.set(1);
+        // close any incoming leader connection (we are leader now)
+        this.leaderConnection?.close();
+        this.leaderConnection = undefined;
+        if (this.useNatsDistribution) {
+            // NATS mode: create leader distributor, no WS discovery needed
+            const conn = await this.natsConnectionManager.connect();
+            this.natsDistLeader = new NatsDistributionLeader(conn, this.natsDistributionSubject, this.logger);
+        }
+        else {
+            // WS mode: start worker discovery
+            this.startWorkerDiscovery();
+            await this.discoverWorkers();
+        }
+        // start the pump with distribution processor
+        this.startPumpAsLeader();
+    }
+    stepDownAsLeader() {
+        if (!this.isLeader)
+            return;
+        this.isLeader = false;
+        clusterMetrics.leaderStatusGauge.set(0);
+        // stop discovery
+        if (this.discoveryInterval) {
+            clearInterval(this.discoveryInterval);
+            this.discoveryInterval = undefined;
+        }
+        // stop lease renewal (separate from election interval)
+        if (this.leaseRenewInterval) {
+            clearInterval(this.leaseRenewInterval);
+            this.leaseRenewInterval = undefined;
+        }
+        // NATS mode: close leader distributor (worker subscription stays)
+        this.natsDistLeader = undefined;
+        // disconnect all WS workers
+        for (const worker of this.workers.values()) {
+            worker.deliveryTracker.rejectAll(new Error("Leader stepping down"));
+            worker.connection.close();
+        }
+        this.workers.clear();
+        clusterMetrics.activeWorkersGauge.set(0);
+        // stop pump
+        if (this.pump) {
+            this.pump.stop();
+            this.pump = undefined;
+        }
+    }
+    startPumpAsLeader() {
+        // create pump with distribution processor that sends events to workers
+        const pumpOptions = {
+            ...this.options,
+            processor: {
+                concurrency: this.workerConcurrency,
+                handler: async (events) => {
+                    await this.distributeEvents(events);
+                },
+                failedHandler: this.workerFailedHandler,
+            },
+        };
+        this.pump = FlowcoreDataPump.create(pumpOptions, this.options.dataSourceOverride);
+        this.pump.start().catch((error) => {
+            this.logger?.error("Pump error in leader mode", { error });
+        });
+    }
+    distributeEvents(events) {
+        if (this.useNatsDistribution) {
+            return this.distributeEventsNats(events);
+        }
+        return this.distributeEventsWs(events);
+    }
+    async distributeEventsNats(events) {
+        if (!this.natsDistLeader) {
+            // no NATS leader distributor - process locally as fallback
+            this.logger?.debug("No NATS leader distributor, processing locally");
+            if (this.workerHandler) {
+                await this.workerHandler(events);
+            }
+            return;
+        }
+        clusterMetrics.eventsDistributedCounter.inc(events.length);
+        try {
+            await this.natsDistLeader.distribute(events);
+            clusterMetrics.workerAcksCounter.inc();
+        }
+        catch (error) {
+            clusterMetrics.workerFailsCounter.inc();
+            // fallback to local processing on NATS failure
+            this.logger?.warn("NATS distribution failed, processing locally", {
+                error: error instanceof Error ? error.message : "unknown",
+            });
+            if (this.workerHandler) {
+                await this.workerHandler(events);
+            }
+        }
+    }
+    async distributeEventsWs(events) {
+        const worker = this.selectWorker();
+        if (!worker) {
+            // no workers available - process locally (single-instance fallback)
+            this.logger?.debug("No workers available, processing locally");
+            if (this.workerHandler) {
+                await this.workerHandler(events);
+            }
+            return;
+        }
+        const deliveryId = crypto.randomUUID();
+        const eventIds = events.map((e) => e.eventId);
+        worker.pendingCount++;
+        clusterMetrics.eventsDistributedCounter.inc(events.length);
+        // send events to worker
+        worker.connection.send({
+            type: "events",
+            deliveryId,
+            events,
+        });
+        // wait for ack
+        try {
+            await worker.deliveryTracker.add(deliveryId, eventIds, DELIVERY_TIMEOUT_MS);
+            worker.pendingCount--;
+            clusterMetrics.workerAcksCounter.inc();
+        }
+        catch (error) {
+            worker.pendingCount--;
+            clusterMetrics.workerFailsCounter.inc();
+            throw error;
+        }
+    }
+    selectWorker() {
+        const available = [];
+        for (const worker of this.workers.values()) {
+            if (worker.pendingCount < this.workerConcurrency && !worker.connection.isClosed) {
+                available.push(worker);
+            }
+        }
+        if (available.length === 0)
+            return null;
+        this.workerRoundRobinIndex = this.workerRoundRobinIndex % available.length;
+        const selected = available[this.workerRoundRobinIndex];
+        this.workerRoundRobinIndex++;
+        return selected;
+    }
+    // #endregion
+    // #region Worker Discovery
+    startWorkerDiscovery() {
+        this.discoveryInterval = setInterval(async () => {
+            if (!this.running || !this.isLeader)
+                return;
+            await this.discoverWorkers();
+        }, DISCOVERY_POLL_INTERVAL_MS);
+    }
+    async discoverWorkers() {
+        try {
+            const instances = await this.coordinator.getInstances(DEFAULT_STALE_THRESHOLD_MS);
+            for (const instance of instances) {
+                // skip self
+                if (instance.instanceId === this.instanceId)
+                    continue;
+                // skip already connected
+                if (this.workers.has(instance.instanceId))
+                    continue;
+                this.connectToWorker(instance.instanceId, instance.address);
+            }
+            // clean up workers that are no longer registered
+            const registeredIds = new Set(instances.map((i) => i.instanceId));
+            for (const [id, worker] of this.workers) {
+                if (!registeredIds.has(id)) {
+                    this.logger?.info("Worker no longer registered, disconnecting", { instanceId: id });
+                    worker.deliveryTracker.rejectAll(new Error("Worker unregistered"));
+                    worker.connection.close();
+                    this.workers.delete(id);
+                }
+            }
+            clusterMetrics.activeWorkersGauge.set(this.workers.size);
+        }
+        catch (error) {
+            this.logger?.warn("Worker discovery failed", { error });
+        }
+    }
+    connectToWorker(instanceId, address) {
+        this.logger?.info("Connecting to worker", { instanceId, address });
+        try {
+            const ws = new dntShim.WebSocket(address);
+            const deliveryTracker = new DeliveryTracker();
+            const workerConn = {
+                instanceId,
+                connection: undefined,
+                deliveryTracker,
+                pendingCount: 0,
+                reconnectAttempts: 0,
+            };
+            ws.onopen = () => {
+                const conn = new WsConnection(ws, {
+                    logger: this.logger,
+                    onMessage: (msg) => this.handleLeaderMessage(msg, instanceId),
+                    onClose: () => this.handleWorkerDisconnect(instanceId),
+                });
+                workerConn.connection = conn;
+                workerConn.reconnectAttempts = 0;
+                this.workers.set(instanceId, workerConn);
+                clusterMetrics.activeWorkersGauge.set(this.workers.size);
+                this.logger?.info("Connected to worker", { instanceId });
+            };
+            ws.onerror = () => {
+                this.logger?.warn("Failed to connect to worker", { instanceId, address });
+                this.scheduleReconnect(instanceId, address, (workerConn.reconnectAttempts || 0) + 1);
+            };
+        }
+        catch (error) {
+            this.logger?.warn("Error connecting to worker", { instanceId, error });
+        }
+    }
+    handleWorkerDisconnect(instanceId) {
+        const worker = this.workers.get(instanceId);
+        if (!worker)
+            return;
+        this.logger?.info("Worker disconnected", { instanceId });
+        // reject all pending deliveries - pump's ack timeout will handle reOpen
+        worker.deliveryTracker.rejectAll(new Error("Worker disconnected"));
+        this.workers.delete(instanceId);
+        clusterMetrics.activeWorkersGauge.set(this.workers.size);
+        // schedule reconnect if still running as leader
+        if (this.running && this.isLeader) {
+            this.scheduleReconnect(instanceId, "", worker.reconnectAttempts + 1);
+        }
+    }
+    scheduleReconnect(instanceId, address, attempts) {
+        if (!this.running || !this.isLeader)
+            return;
+        const delay = Math.min(RECONNECT_BASE_MS * Math.pow(2, attempts - 1), RECONNECT_MAX_MS);
+        this.logger?.debug("Scheduling reconnect", { instanceId, delay, attempts });
+        setTimeout(async () => {
+            if (!this.running || !this.isLeader)
+                return;
+            // re-discover address from DB if we don't have it
+            let resolvedAddress = address;
+            if (!resolvedAddress) {
+                try {
+                    const instances = await this.coordinator.getInstances(DEFAULT_STALE_THRESHOLD_MS);
+                    const instance = instances.find((i) => i.instanceId === instanceId);
+                    if (!instance)
+                        return; // worker gone, don't reconnect
+                    resolvedAddress = instance.address;
+                }
+                catch {
+                    return;
+                }
+            }
+            if (!this.workers.has(instanceId)) {
+                this.connectToWorker(instanceId, resolvedAddress);
+            }
+        }, delay);
+    }
+    // #endregion
+    // #region Message Handlers
+    // leader receives ack/fail from workers
+    handleLeaderMessage(msg, workerInstanceId) {
+        const worker = this.workers.get(workerInstanceId);
+        if (!worker)
+            return;
+        if (msg.type === "ack") {
+            worker.deliveryTracker.ack(msg.deliveryId);
+        }
+        else if (msg.type === "fail") {
+            worker.deliveryTracker.fail(msg.deliveryId);
+        }
+    }
+    // worker receives events from leader
+    handleWorkerMessage(msg, conn) {
+        if (msg.type !== "events")
+            return;
+        const { deliveryId, events } = msg;
+        this.logger?.debug("Received events from leader", { deliveryId, count: events.length });
+        // process events using user's handler
+        void (async () => {
+            try {
+                if (this.workerHandler) {
+                    await this.workerHandler(events);
+                }
+                conn.send({
+                    type: "ack",
+                    deliveryId,
+                    eventIds: events.map((e) => e.eventId),
+                });
+            }
+            catch (error) {
+                this.logger?.error("Worker failed to process events", { deliveryId, error });
+                conn.send({
+                    type: "fail",
+                    deliveryId,
+                    eventIds: events.map((e) => e.eventId),
+                });
+            }
+        })();
+    }
+}