@flowcore/pathways 0.16.3 → 0.17.0

package/script/pathways/cluster/cluster-manager.js

@@ -0,0 +1,650 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClusterManager = void 0;
+const dntShim = __importStar(require("../../_dnt.shims.js"));
+const logger_js_1 = require("../logger.js");
+const DEFAULT_LEASE_TTL_MS = 30_000;
+const DEFAULT_LEASE_RENEW_INTERVAL_MS = 10_000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 5_000;
+const DEFAULT_STALE_THRESHOLD_MS = 15_000;
+const DEFAULT_DELIVERY_TIMEOUT_MS = 30_000;
+const LEASE_KEY = "pathway-cluster-leader";
+/**
+ * Creates a default transport using Deno APIs.
+ * Uses runtime detection to avoid compile-time dependency on Deno types.
+ */
+function createDefaultTransport() {
+    // deno-lint-ignore no-explicit-any
+    const D = dntShim.dntGlobalThis.Deno;
+    if (!D?.serve) {
+        throw new Error("Default cluster transport requires Deno. Provide a custom ClusterTransport via options.transport for Node.js.");
+    }
+    return {
+        startServer(port, onConnection) {
+            const server = D.serve({ port, hostname: "0.0.0.0" }, (req) => {
+                if (req.headers.get("upgrade")?.toLowerCase() !== "websocket") {
+                    return new Response("Expected WebSocket", { status: 426 });
+                }
+                const { socket, response } = D.upgradeWebSocket(req);
+                socket.onopen = () => onConnection(socket);
+                return response;
+            });
+            return Promise.resolve({ shutdown: () => server.shutdown() });
+        },
+        connect(address) {
+            return new dntShim.WebSocket(address);
+        },
+    };
+}
+/**
+ * ClusterManager handles distributed event processing via leader election and WS workers.
+ *
+ * Lifecycle: register → heartbeat loop → leader election loop
+ * - Leader: opens WS connections to workers, distributes events round-robin
+ * - Worker: accepts WS from leader, processes events, sends ack/fail
+ * - Fallback: leader processes locally when no workers available
+ */
+class ClusterManager {
+    constructor(options, logger) {
+        Object.defineProperty(this, "coordinator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "transport", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "instanceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "advertisedAddress", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "port", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaseTtlMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaseRenewIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "heartbeatIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "staleThresholdMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "deliveryTimeoutMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workerConcurrency", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "role", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "unknown"
+        });
+        Object.defineProperty(this, "running", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "heartbeatTimer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "leaseTimer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "deliveryTimeoutTimer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        // Leader state
+        Object.defineProperty(this, "workerConnections", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "pendingDeliveries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "workerAddresses", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "roundRobinIndex", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        // Worker state
+        Object.defineProperty(this, "wsServer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "leaderConnection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "eventHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        this.coordinator = options.coordinator;
+        this.transport = options.transport ?? createDefaultTransport();
+        this.instanceId = crypto.randomUUID();
+        this.advertisedAddress = options.advertisedAddress;
+        this.port = options.port;
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
+        this.leaseRenewIntervalMs = options.leaseRenewIntervalMs ?? DEFAULT_LEASE_RENEW_INTERVAL_MS;
+        this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
+        this.staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
+        this.deliveryTimeoutMs = options.deliveryTimeoutMs ?? DEFAULT_DELIVERY_TIMEOUT_MS;
+        this.workerConcurrency = options.workerConcurrency ?? 5;
+        this.logger = logger ?? new logger_js_1.NoopLogger();
+    }
+    /**
+     * Set the handler that processes events locally (used by both leader fallback and workers)
+     */
+    setEventHandler(handler) {
+        this.eventHandler = handler;
+    }
+    /**
+     * Start the cluster: register instance, begin heartbeat, attempt leader election
+     */
+    async start() {
+        if (this.running)
+            return;
+        this.running = true;
+        this.logger.info("Starting cluster manager", {
+            instanceId: this.instanceId,
+            advertisedAddress: this.advertisedAddress,
+        });
+        // Register this instance
+        await this.coordinator.register(this.instanceId, this.advertisedAddress);
+        // Start WS server for accepting connections (both leader and worker need this)
+        await this.startWsServer();
+        // Start heartbeat loop
+        this.heartbeatTimer = setInterval(async () => {
+            try {
+                await this.coordinator.heartbeat(this.instanceId);
+            }
+            catch (err) {
+                this.logger.error("Heartbeat failed", err instanceof Error ? err : new Error(String(err)));
+            }
+        }, this.heartbeatIntervalMs);
+        // Start leader election loop
+        await this.tryAcquireLease();
+        this.leaseTimer = setInterval(async () => {
+            try {
+                await this.leaseLoop();
+            }
+            catch (err) {
+                this.logger.error("Lease loop error", err instanceof Error ? err : new Error(String(err)));
+            }
+        }, this.leaseRenewIntervalMs);
+        // Start delivery timeout checker
+        this.deliveryTimeoutTimer = setInterval(() => {
+            this.checkDeliveryTimeouts();
+        }, 5_000);
+    }
+    /**
+     * Stop the cluster: release lease, unregister, close connections
+     */
+    async stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.logger.info("Stopping cluster manager", { instanceId: this.instanceId });
+        if (this.heartbeatTimer)
+            clearInterval(this.heartbeatTimer);
+        if (this.leaseTimer)
+            clearInterval(this.leaseTimer);
+        if (this.deliveryTimeoutTimer)
+            clearInterval(this.deliveryTimeoutTimer);
+        // Release lease if leader
+        if (this.role === "leader") {
+            try {
+                await this.coordinator.releaseLease(this.instanceId, LEASE_KEY);
+            }
+            catch (err) {
+                this.logger.error("Error releasing lease", err instanceof Error ? err : new Error(String(err)));
+            }
+        }
+        // Close worker connections (leader side)
+        for (const [, ws] of this.workerConnections) {
+            try {
+                ws.close();
+            }
+            catch {
+                // ignore close errors
+            }
+        }
+        this.workerConnections.clear();
+        // Close leader connection (worker side)
+        if (this.leaderConnection) {
+            try {
+                this.leaderConnection.close();
+            }
+            catch {
+                // ignore
+            }
+            this.leaderConnection = null;
+        }
+        // Stop WS server
+        if (this.wsServer) {
+            try {
+                await this.wsServer.shutdown();
+            }
+            catch {
+                // ignore
+            }
+            this.wsServer = null;
+        }
+        // Reject pending deliveries
+        for (const [, delivery] of this.pendingDeliveries) {
+            delivery.reject(new Error("Cluster manager stopped"));
+        }
+        this.pendingDeliveries.clear();
+        // Unregister
+        try {
+            await this.coordinator.unregister(this.instanceId);
+        }
+        catch (err) {
+            this.logger.error("Error unregistering", err instanceof Error ? err : new Error(String(err)));
+        }
+        this.role = "unknown";
+    }
+    /**
+     * Process an event through the cluster.
+     * - Leader: distribute to a worker or process locally if no workers
+     * - Worker: should not call this directly (receives events via WS)
+     */
+    async processEvent(pathway, event) {
+        if (this.role !== "leader") {
+            this.logger.warn("processEvent called on non-leader instance, processing locally", {
+                instanceId: this.instanceId,
+                role: this.role,
+            });
+            await this.processLocally(pathway, event);
+            return;
+        }
+        // Try to distribute to a worker
+        const workerAddress = this.getNextWorker();
+        if (workerAddress) {
+            try {
+                await this.distributeToWorker(workerAddress, pathway, event);
+                return;
+            }
+            catch (err) {
+                this.logger.warn("Failed to distribute to worker, processing locally", {
+                    worker: workerAddress,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        // No workers available or distribution failed — process locally
+        await this.processLocally(pathway, event);
+    }
+    get isLeader() {
+        return this.role === "leader";
+    }
+    get isWorker() {
+        return this.role === "worker";
+    }
+    get currentRole() {
+        return this.role;
+    }
+    get isRunning() {
+        return this.running;
+    }
+    get currentInstanceId() {
+        return this.instanceId;
+    }
+    // --- Private: Leader Election ---
+    async tryAcquireLease() {
+        const acquired = await this.coordinator.acquireLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+        if (acquired) {
+            if (this.role !== "leader") {
+                this.logger.info("Acquired leader lease", { instanceId: this.instanceId });
+                this.role = "leader";
+                await this.onBecomeLeader();
+            }
+        }
+        else {
+            if (this.role !== "worker") {
+                this.logger.info("Could not acquire lease, becoming worker", { instanceId: this.instanceId });
+                this.role = "worker";
+            }
+        }
+    }
+    async leaseLoop() {
+        if (this.role === "leader") {
+            const renewed = await this.coordinator.renewLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+            if (!renewed) {
+                this.logger.warn("Lost leader lease", { instanceId: this.instanceId });
+                this.role = "worker";
+                this.cleanupLeaderState();
+            }
+            else {
+                await this.refreshWorkers();
+            }
+        }
+        else {
+            await this.tryAcquireLease();
+        }
+    }
+    async onBecomeLeader() {
+        await this.refreshWorkers();
+    }
+    async refreshWorkers() {
+        const instances = await this.coordinator.getInstances(this.staleThresholdMs);
+        const newWorkers = instances
+            .filter((i) => i.instanceId !== this.instanceId)
+            .map((i) => i.address);
+        const currentAddresses = new Set(this.workerConnections.keys());
+        const newAddresses = new Set(newWorkers);
+        // Disconnect from removed workers
+        for (const addr of currentAddresses) {
+            if (!newAddresses.has(addr)) {
+                const ws = this.workerConnections.get(addr);
+                if (ws) {
+                    try {
+                        ws.close();
+                    }
+                    catch {
+                        // ignore
+                    }
+                }
+                this.workerConnections.delete(addr);
+            }
+        }
+        // Connect to new workers
+        for (const addr of newAddresses) {
+            if (!currentAddresses.has(addr)) {
+                this.connectToWorker(addr);
+            }
+        }
+        this.workerAddresses = [...newAddresses];
+        this.logger.debug("Refreshed worker list", { workers: this.workerAddresses });
+    }
+    cleanupLeaderState() {
+        for (const [, ws] of this.workerConnections) {
+            try {
+                ws.close();
+            }
+            catch {
+                // ignore
+            }
+        }
+        this.workerConnections.clear();
+        this.workerAddresses = [];
+        this.roundRobinIndex = 0;
+    }
+    // --- Private: WS Server (accepts connections from leader) ---
+    async startWsServer() {
+        this.wsServer = await this.transport.startServer(this.port, (socket) => {
+            this.logger.debug("WS connection opened from leader");
+            this.leaderConnection = socket;
+            socket.onmessage = async (event) => {
+                try {
+                    const msg = JSON.parse(event.data);
+                    await this.handleWorkerMessage(socket, msg);
+                }
+                catch (err) {
+                    this.logger.error("Error handling WS message", err instanceof Error ? err : new Error(String(err)));
+                }
+            };
+            socket.onclose = () => {
+                this.logger.debug("WS connection closed from leader");
+                if (this.leaderConnection === socket) {
+                    this.leaderConnection = null;
+                }
+            };
+            socket.onerror = (err) => {
+                this.logger.error("WS error", new Error(String(err)));
+            };
+        });
+        this.logger.info("WS server started", { port: this.port });
+    }
+    async handleWorkerMessage(socket, msg) {
+        switch (msg.type) {
+            case "events": {
+                const ackEventIds = [];
+                const failEventIds = [];
+                for (const event of msg.events) {
+                    const pathway = `${event.flowType}/${event.eventType}`;
+                    try {
+                        await this.processLocally(pathway, event);
+                        ackEventIds.push(event.eventId);
+                    }
+                    catch {
+                        failEventIds.push(event.eventId);
+                    }
+                }
+                if (ackEventIds.length > 0) {
+                    const ack = {
+                        type: "ack",
+                        deliveryId: msg.deliveryId,
+                        eventIds: ackEventIds,
+                    };
+                    socket.send(JSON.stringify(ack));
+                }
+                if (failEventIds.length > 0) {
+                    const fail = {
+                        type: "fail",
+                        deliveryId: msg.deliveryId,
+                        eventIds: failEventIds,
+                    };
+                    socket.send(JSON.stringify(fail));
+                }
+                break;
+            }
+            case "ping": {
+                socket.send(JSON.stringify({ type: "pong" }));
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    // --- Private: WS Client (leader connects to workers) ---
+    connectToWorker(address) {
+        try {
+            const ws = this.transport.connect(address);
+            ws.onopen = () => {
+                this.logger.info("Connected to worker", { address });
+                this.workerConnections.set(address, ws);
+            };
+            ws.onmessage = (event) => {
+                try {
+                    const msg = JSON.parse(event.data);
+                    this.handleLeaderMessage(address, msg);
+                }
+                catch (err) {
+                    this.logger.error("Error handling worker response", err instanceof Error ? err : new Error(String(err)));
+                }
+            };
+            ws.onclose = () => {
+                this.logger.info("Disconnected from worker", { address });
+                this.workerConnections.delete(address);
+                this.workerAddresses = this.workerAddresses.filter((a) => a !== address);
+            };
+            ws.onerror = (err) => {
+                this.logger.error("Worker WS error", new Error(String(err)), { address });
+                this.workerConnections.delete(address);
+            };
+        }
+        catch (err) {
+            this.logger.error("Failed to connect to worker", err instanceof Error ? err : new Error(String(err)), { address });
+        }
+    }
+    handleLeaderMessage(workerAddress, msg) {
+        switch (msg.type) {
+            case "ack": {
+                const delivery = this.pendingDeliveries.get(msg.deliveryId);
+                if (delivery) {
+                    this.pendingDeliveries.delete(msg.deliveryId);
+                    delivery.resolve(msg.eventIds);
+                }
+                break;
+            }
+            case "fail": {
+                const delivery = this.pendingDeliveries.get(msg.deliveryId);
+                if (delivery) {
+                    this.pendingDeliveries.delete(msg.deliveryId);
+                    delivery.reject(new Error(`Worker ${workerAddress} failed to process events: ${msg.eventIds.join(", ")}`));
+                }
+                break;
+            }
+            case "pong": {
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    // --- Private: Event Distribution ---
+    getNextWorker() {
+        const connectedWorkers = this.workerAddresses.filter((addr) => {
+            const ws = this.workerConnections.get(addr);
+            return ws && ws.readyState === dntShim.WebSocket.OPEN;
+        });
+        if (connectedWorkers.length === 0)
+            return null;
+        const worker = connectedWorkers[this.roundRobinIndex % connectedWorkers.length];
+        this.roundRobinIndex = (this.roundRobinIndex + 1) % connectedWorkers.length;
+        return worker;
+    }
+    distributeToWorker(workerAddress, _pathway, event) {
+        const ws = this.workerConnections.get(workerAddress);
+        if (!ws || ws.readyState !== dntShim.WebSocket.OPEN) {
+            throw new Error(`Worker ${workerAddress} not connected`);
+        }
+        const deliveryId = crypto.randomUUID();
+        return new Promise((resolve, reject) => {
+            const delivery = {
+                deliveryId,
+                events: [event],
+                workerAddress,
+                sentAt: Date.now(),
+                resolve: () => resolve(),
+                reject,
+            };
+            this.pendingDeliveries.set(deliveryId, delivery);
+            const msg = {
+                type: "events",
+                deliveryId,
+                events: [event],
+            };
+            try {
+                ws.send(JSON.stringify(msg));
+            }
+            catch (err) {
+                this.pendingDeliveries.delete(deliveryId);
+                reject(err instanceof Error ? err : new Error(String(err)));
+            }
+        });
+    }
+    async processLocally(pathway, event) {
+        if (!this.eventHandler) {
+            throw new Error("No event handler set on ClusterManager");
+        }
+        await this.eventHandler(pathway, event);
+    }
+    checkDeliveryTimeouts() {
+        const now = Date.now();
+        for (const [id, delivery] of this.pendingDeliveries) {
+            if (now - delivery.sentAt > this.deliveryTimeoutMs) {
+                this.pendingDeliveries.delete(id);
+                delivery.reject(new Error(`Delivery ${id} to ${delivery.workerAddress} timed out`));
+            }
+        }
+    }
+}
+exports.ClusterManager = ClusterManager;

package/script/pathways/cluster/index.d.ts

@@ -0,0 +1,4 @@
+export type { ClusterRole, ClusterSocket, ClusterTransport, PathwayClusterOptions, PathwayCoordinator, PendingDelivery, WsAckMessage, WsEventsMessage, WsFailMessage, WsMessage, WsPingMessage, WsPongMessage, } from "./types.js";
+export { ClusterManager } from "./cluster-manager.js";
+export { createPostgresPathwayCoordinator, PostgresPathwayCoordinator } from "./postgres-coordinator.js";
+//# sourceMappingURL=index.d.ts.map

package/script/pathways/cluster/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/pathways/cluster/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,eAAe,EACf,aAAa,EACb,SAAS,EACT,aAAa,EACb,aAAa,GACd,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,gCAAgC,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA"}

package/script/pathways/cluster/index.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostgresPathwayCoordinator = exports.createPostgresPathwayCoordinator = exports.ClusterManager = void 0;
+var cluster_manager_js_1 = require("./cluster-manager.js");
+Object.defineProperty(exports, "ClusterManager", { enumerable: true, get: function () { return cluster_manager_js_1.ClusterManager; } });
+var postgres_coordinator_js_1 = require("./postgres-coordinator.js");
+Object.defineProperty(exports, "createPostgresPathwayCoordinator", { enumerable: true, get: function () { return postgres_coordinator_js_1.createPostgresPathwayCoordinator; } });
+Object.defineProperty(exports, "PostgresPathwayCoordinator", { enumerable: true, get: function () { return postgres_coordinator_js_1.PostgresPathwayCoordinator; } });