@flowcore/data-pump 0.16.5 → 0.17.0

package/script/data-pump/data-pump-cluster.js

@@ -0,0 +1,585 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FlowcoreDataPumpCluster = void 0;
+const dntShim = __importStar(require("../_dnt.shims.js"));
+const data_pump_js_1 = require("./data-pump.js");
+const metrics_js_1 = require("./metrics.js");
+const ws_protocol_js_1 = require("./ws-protocol.js");
+const LEASE_KEY = "flowcore-data-pump-leader";
+const DISCOVERY_POLL_INTERVAL_MS = 10_000;
+const DEFAULT_LEASE_TTL_MS = 30_000;
+const DEFAULT_LEASE_RENEW_INTERVAL_MS = 10_000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 5_000;
+const DEFAULT_STALE_THRESHOLD_MS = 30_000;
+const DEFAULT_WORKER_CONCURRENCY = 1;
+const RECONNECT_BASE_MS = 1_000;
+const RECONNECT_MAX_MS = 30_000;
+const DELIVERY_TIMEOUT_MS = 30_000;
+class FlowcoreDataPumpCluster {
+    constructor(options) {
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+        Object.defineProperty(this, "instanceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "coordinator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaseTtlMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaseRenewIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "heartbeatIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workerConcurrency", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "running", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "isLeader", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "pump", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "leaderConnection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // leader state
+        Object.defineProperty(this, "workers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "workerRoundRobinIndex", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "leaseRenewInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "discoveryInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // common state
+        Object.defineProperty(this, "heartbeatInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "electionInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // worker state - handler for processing events received from leader
+        Object.defineProperty(this, "workerHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workerFailedHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.instanceId = crypto.randomUUID();
+        this.coordinator = options.coordinator;
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
+        this.leaseRenewIntervalMs = options.leaseRenewIntervalMs ?? DEFAULT_LEASE_RENEW_INTERVAL_MS;
+        this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
+        this.workerConcurrency = options.workerConcurrency ?? DEFAULT_WORKER_CONCURRENCY;
+        this.logger = options.logger;
+        // capture user's processor for worker mode
+        this.workerHandler = options.processor?.handler;
+        this.workerFailedHandler = options.processor?.failedHandler;
+    }
+    get isRunning() {
+        return this.running;
+    }
+    get isLeaderInstance() {
+        return this.isLeader;
+    }
+    get activeWorkerCount() {
+        return this.workers.size;
+    }
+    get id() {
+        return this.instanceId;
+    }
+    /**
+     * Handle an incoming WebSocket connection from a peer.
+     * Call this from your HTTP server's WebSocket upgrade handler.
+     *
+     * Example (Deno):
+     * ```typescript
+     * Deno.serve({ port: 8080 }, (req) => {
+     *   if (req.headers.get("upgrade") === "websocket") {
+     *     const { socket, response } = Deno.upgradeWebSocket(req)
+     *     cluster.handleConnection(socket)
+     *     return response
+     *   }
+     *   return new Response("Not found", { status: 404 })
+     * })
+     * ```
+     *
+     * Example (Node.js with ws):
+     * ```typescript
+     * import { WebSocketServer } from "ws"
+     * const wss = new WebSocketServer({ port: 8080 })
+     * wss.on("connection", (ws) => cluster.handleConnection(ws))
+     * ```
+     */
+    handleConnection(ws) {
+        this.logger?.debug("Incoming WS connection");
+        const conn = new ws_protocol_js_1.WsConnection(ws, {
+            logger: this.logger,
+            onMessage: (msg) => this.handleWorkerMessage(msg, conn),
+            onClose: () => {
+                this.logger?.info("Leader connection closed");
+                this.leaderConnection = undefined;
+            },
+        });
+        this.leaderConnection = conn;
+    }
+    async start() {
+        if (this.running) {
+            throw new Error("Cluster already running");
+        }
+        this.running = true;
+        this.logger?.info("Starting cluster instance", { instanceId: this.instanceId });
+        // Step 1: Register in DB
+        await this.coordinator.register(this.instanceId, this.options.advertisedAddress);
+        // Step 2: Start heartbeat
+        this.startHeartbeat();
+        // Step 3: Start leader election loop
+        this.startElectionLoop();
+    }
+    async stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.logger?.info("Stopping cluster instance", { instanceId: this.instanceId });
+        // stop election + heartbeat
+        if (this.electionInterval) {
+            clearInterval(this.electionInterval);
+            this.electionInterval = undefined;
+        }
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = undefined;
+        }
+        // stop leader duties
+        this.stepDownAsLeader();
+        // close worker connection to leader
+        this.leaderConnection?.close();
+        this.leaderConnection = undefined;
+        // unregister
+        try {
+            await this.coordinator.releaseLease(this.instanceId, LEASE_KEY);
+        }
+        catch {
+            // ignore
+        }
+        try {
+            await this.coordinator.unregister(this.instanceId);
+        }
+        catch {
+            // ignore
+        }
+        metrics_js_1.clusterMetrics.leaderStatusGauge.set(0);
+        metrics_js_1.clusterMetrics.activeWorkersGauge.set(0);
+        this.logger?.info("Cluster instance stopped");
+    }
+    // #region Heartbeat
+    startHeartbeat() {
+        this.heartbeatInterval = setInterval(async () => {
+            if (!this.running)
+                return;
+            try {
+                await this.coordinator.heartbeat(this.instanceId);
+            }
+            catch (error) {
+                this.logger?.warn("Heartbeat failed", { error });
+            }
+        }, this.heartbeatIntervalMs);
+    }
+    // #endregion
+    // #region Leader Election
+    startElectionLoop() {
+        // attempt immediately
+        void this.attemptElection();
+        this.electionInterval = setInterval(async () => {
+            if (!this.running)
+                return;
+            if (this.isLeader) {
+                // renew lease
+                try {
+                    const renewed = await this.coordinator.renewLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+                    if (!renewed) {
+                        this.logger?.warn("Lease renewal failed, stepping down");
+                        this.stepDownAsLeader();
+                    }
+                }
+                catch (error) {
+                    this.logger?.error("Lease renewal error, stepping down", { error });
+                    this.stepDownAsLeader();
+                }
+            }
+            else {
+                await this.attemptElection();
+            }
+        }, this.leaseRenewIntervalMs);
+    }
+    async attemptElection() {
+        try {
+            const acquired = await this.coordinator.acquireLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+            if (acquired && !this.isLeader) {
+                this.logger?.info("Won leader election", { instanceId: this.instanceId });
+                await this.becomeLeader();
+            }
+        }
+        catch (error) {
+            this.logger?.debug("Election attempt failed", { error });
+        }
+    }
+    // #endregion
+    // #region Leader Mode
+    async becomeLeader() {
+        this.isLeader = true;
+        metrics_js_1.clusterMetrics.leaderStatusGauge.set(1);
+        // close any incoming leader connection (we are leader now)
+        this.leaderConnection?.close();
+        this.leaderConnection = undefined;
+        // start worker discovery
+        this.startWorkerDiscovery();
+        // initial discovery
+        await this.discoverWorkers();
+        // start the pump with distribution processor
+        this.startPumpAsLeader();
+    }
+    stepDownAsLeader() {
+        if (!this.isLeader)
+            return;
+        this.isLeader = false;
+        metrics_js_1.clusterMetrics.leaderStatusGauge.set(0);
+        // stop discovery
+        if (this.discoveryInterval) {
+            clearInterval(this.discoveryInterval);
+            this.discoveryInterval = undefined;
+        }
+        // stop lease renewal (separate from election interval)
+        if (this.leaseRenewInterval) {
+            clearInterval(this.leaseRenewInterval);
+            this.leaseRenewInterval = undefined;
+        }
+        // disconnect all workers
+        for (const worker of this.workers.values()) {
+            worker.deliveryTracker.rejectAll(new Error("Leader stepping down"));
+            worker.connection.close();
+        }
+        this.workers.clear();
+        metrics_js_1.clusterMetrics.activeWorkersGauge.set(0);
+        // stop pump
+        if (this.pump) {
+            this.pump.stop();
+            this.pump = undefined;
+        }
+    }
+    startPumpAsLeader() {
+        // create pump with distribution processor that sends events to workers
+        const pumpOptions = {
+            ...this.options,
+            processor: {
+                concurrency: this.workerConcurrency,
+                handler: async (events) => {
+                    await this.distributeEvents(events);
+                },
+                failedHandler: this.workerFailedHandler,
+            },
+        };
+        this.pump = data_pump_js_1.FlowcoreDataPump.create(pumpOptions);
+        this.pump.start().catch((error) => {
+            this.logger?.error("Pump error in leader mode", { error });
+        });
+    }
+    async distributeEvents(events) {
+        const worker = this.selectWorker();
+        if (!worker) {
+            // no workers available - process locally (single-instance fallback)
+            this.logger?.debug("No workers available, processing locally");
+            if (this.workerHandler) {
+                await this.workerHandler(events);
+            }
+            return;
+        }
+        const deliveryId = crypto.randomUUID();
+        const eventIds = events.map((e) => e.eventId);
+        worker.pendingCount++;
+        metrics_js_1.clusterMetrics.eventsDistributedCounter.inc(events.length);
+        // send events to worker
+        worker.connection.send({
+            type: "events",
+            deliveryId,
+            events,
+        });
+        // wait for ack
+        try {
+            await worker.deliveryTracker.add(deliveryId, eventIds, DELIVERY_TIMEOUT_MS);
+            worker.pendingCount--;
+            metrics_js_1.clusterMetrics.workerAcksCounter.inc();
+        }
+        catch (error) {
+            worker.pendingCount--;
+            metrics_js_1.clusterMetrics.workerFailsCounter.inc();
+            throw error;
+        }
+    }
+    selectWorker() {
+        const available = [];
+        for (const worker of this.workers.values()) {
+            if (worker.pendingCount < this.workerConcurrency && !worker.connection.isClosed) {
+                available.push(worker);
+            }
+        }
+        if (available.length === 0)
+            return null;
+        this.workerRoundRobinIndex = this.workerRoundRobinIndex % available.length;
+        const selected = available[this.workerRoundRobinIndex];
+        this.workerRoundRobinIndex++;
+        return selected;
+    }
+    // #endregion
+    // #region Worker Discovery
+    startWorkerDiscovery() {
+        this.discoveryInterval = setInterval(async () => {
+            if (!this.running || !this.isLeader)
+                return;
+            await this.discoverWorkers();
+        }, DISCOVERY_POLL_INTERVAL_MS);
+    }
+    async discoverWorkers() {
+        try {
+            const instances = await this.coordinator.getInstances(DEFAULT_STALE_THRESHOLD_MS);
+            for (const instance of instances) {
+                // skip self
+                if (instance.instanceId === this.instanceId)
+                    continue;
+                // skip already connected
+                if (this.workers.has(instance.instanceId))
+                    continue;
+                this.connectToWorker(instance.instanceId, instance.address);
+            }
+            // clean up workers that are no longer registered
+            const registeredIds = new Set(instances.map((i) => i.instanceId));
+            for (const [id, worker] of this.workers) {
+                if (!registeredIds.has(id)) {
+                    this.logger?.info("Worker no longer registered, disconnecting", { instanceId: id });
+                    worker.deliveryTracker.rejectAll(new Error("Worker unregistered"));
+                    worker.connection.close();
+                    this.workers.delete(id);
+                }
+            }
+            metrics_js_1.clusterMetrics.activeWorkersGauge.set(this.workers.size);
+        }
+        catch (error) {
+            this.logger?.warn("Worker discovery failed", { error });
+        }
+    }
+    connectToWorker(instanceId, address) {
+        this.logger?.info("Connecting to worker", { instanceId, address });
+        try {
+            const ws = new dntShim.WebSocket(address);
+            const deliveryTracker = new ws_protocol_js_1.DeliveryTracker();
+            const workerConn = {
+                instanceId,
+                connection: undefined,
+                deliveryTracker,
+                pendingCount: 0,
+                reconnectAttempts: 0,
+            };
+            ws.onopen = () => {
+                const conn = new ws_protocol_js_1.WsConnection(ws, {
+                    logger: this.logger,
+                    onMessage: (msg) => this.handleLeaderMessage(msg, instanceId),
+                    onClose: () => this.handleWorkerDisconnect(instanceId),
+                });
+                workerConn.connection = conn;
+                workerConn.reconnectAttempts = 0;
+                this.workers.set(instanceId, workerConn);
+                metrics_js_1.clusterMetrics.activeWorkersGauge.set(this.workers.size);
+                this.logger?.info("Connected to worker", { instanceId });
+            };
+            ws.onerror = () => {
+                this.logger?.warn("Failed to connect to worker", { instanceId, address });
+                this.scheduleReconnect(instanceId, address, (workerConn.reconnectAttempts || 0) + 1);
+            };
+        }
+        catch (error) {
+            this.logger?.warn("Error connecting to worker", { instanceId, error });
+        }
+    }
+    handleWorkerDisconnect(instanceId) {
+        const worker = this.workers.get(instanceId);
+        if (!worker)
+            return;
+        this.logger?.info("Worker disconnected", { instanceId });
+        // reject all pending deliveries - pump's ack timeout will handle reOpen
+        worker.deliveryTracker.rejectAll(new Error("Worker disconnected"));
+        this.workers.delete(instanceId);
+        metrics_js_1.clusterMetrics.activeWorkersGauge.set(this.workers.size);
+        // schedule reconnect if still running as leader
+        if (this.running && this.isLeader) {
+            this.scheduleReconnect(instanceId, "", worker.reconnectAttempts + 1);
+        }
+    }
+    scheduleReconnect(instanceId, address, attempts) {
+        if (!this.running || !this.isLeader)
+            return;
+        const delay = Math.min(RECONNECT_BASE_MS * Math.pow(2, attempts - 1), RECONNECT_MAX_MS);
+        this.logger?.debug("Scheduling reconnect", { instanceId, delay, attempts });
+        setTimeout(async () => {
+            if (!this.running || !this.isLeader)
+                return;
+            // re-discover address from DB if we don't have it
+            let resolvedAddress = address;
+            if (!resolvedAddress) {
+                try {
+                    const instances = await this.coordinator.getInstances(DEFAULT_STALE_THRESHOLD_MS);
+                    const instance = instances.find((i) => i.instanceId === instanceId);
+                    if (!instance)
+                        return; // worker gone, don't reconnect
+                    resolvedAddress = instance.address;
+                }
+                catch {
+                    return;
+                }
+            }
+            if (!this.workers.has(instanceId)) {
+                this.connectToWorker(instanceId, resolvedAddress);
+            }
+        }, delay);
+    }
+    // #endregion
+    // #region Message Handlers
+    // leader receives ack/fail from workers
+    handleLeaderMessage(msg, workerInstanceId) {
+        const worker = this.workers.get(workerInstanceId);
+        if (!worker)
+            return;
+        if (msg.type === "ack") {
+            worker.deliveryTracker.ack(msg.deliveryId);
+        }
+        else if (msg.type === "fail") {
+            worker.deliveryTracker.fail(msg.deliveryId);
+        }
+    }
+    // worker receives events from leader
+    handleWorkerMessage(msg, conn) {
+        if (msg.type !== "events")
+            return;
+        const { deliveryId, events } = msg;
+        this.logger?.debug("Received events from leader", { deliveryId, count: events.length });
+        // process events using user's handler
+        void (async () => {
+            try {
+                if (this.workerHandler) {
+                    await this.workerHandler(events);
+                }
+                conn.send({
+                    type: "ack",
+                    deliveryId,
+                    eventIds: events.map((e) => e.eventId),
+                });
+            }
+            catch (error) {
+                this.logger?.error("Worker failed to process events", { deliveryId, error });
+                conn.send({
+                    type: "fail",
+                    deliveryId,
+                    eventIds: events.map((e) => e.eventId),
+                });
+            }
+        })();
+    }
+}
+exports.FlowcoreDataPumpCluster = FlowcoreDataPumpCluster;

package/script/data-pump/data-pump.d.ts

@@ -1,4 +1,4 @@
-import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
 import { FlowcoreDataSource } from "./data-source.js";
 import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpProcessor, FlowcoreDataPumpState, FlowcoreDataPumpStateManager, FlowcoreLogger } from "./types.js";
 interface FlowcoreDataPumpNotifierNatsOptions {

package/script/data-pump/data-source.d.ts

@@ -2,7 +2,7 @@
  * @module data-source
  * @description Provides functionality to interact with Flowcore data sources
  */
-import { type EventListOutput, type FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import { type EventListOutput, type FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
 import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpState } from "./types.js";
 /**
  * Options for configuring a Flowcore data source

package/script/data-pump/data-source.js

@@ -5,7 +5,7 @@ exports.FlowcoreDataSource = void 0;
  * @module data-source
  * @description Provides functionality to interact with Flowcore data sources
  */
-const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js");
+const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js");
 const flowcore_client_js_1 = require("./flowcore-client.js");
 /**
  * Provides methods to interact with a Flowcore data source

package/script/data-pump/flowcore-client.d.ts

@@ -1,4 +1,4 @@
-import { type Command, FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import { type Command, FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
 import type { FlowcoreDataPumpAuth } from "./types.js";
 declare class FlowcoreClientWithMetrics extends FlowcoreClient {
     execute<Input, Output>(command: Command<Input, Output>, directMode?: boolean): Promise<Output>;

package/script/data-pump/flowcore-client.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getFlowcoreClient = getFlowcoreClient;
-const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js");
+const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js");
 const metrics_js_1 = require("./metrics.js");
 class FlowcoreClientWithMetrics extends mod_js_1.FlowcoreClient {
     execute(command, directMode) {

package/script/data-pump/metrics.d.ts

@@ -9,4 +9,11 @@ export declare const metrics: {
     eventsPulledSizeBytesCounter: PromClient.Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
     sdkCommandsCounter: PromClient.Counter<"command">;
 };
+export declare const clusterMetrics: {
+    activeWorkersGauge: PromClient.Gauge<string>;
+    leaderStatusGauge: PromClient.Gauge<string>;
+    eventsDistributedCounter: PromClient.Counter<string>;
+    workerAcksCounter: PromClient.Counter<string>;
+    workerFailsCounter: PromClient.Counter<string>;
+};
 //# sourceMappingURL=metrics.d.ts.map

package/script/data-pump/metrics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/data-pump/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,UAA8B,MAAM,aAAa,CAAA;AAExD,eAAO,MAAM,oBAAoB,EAAE,UAAU,CAAC,QAAQ,CAAC,0CAA0C,CACpF,CAAA;AAoDb,eAAO,MAAM,OAAO;;;;;;;;CAQnB,CAAA"}
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/data-pump/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,UAA8B,MAAM,aAAa,CAAA;AAExD,eAAO,MAAM,oBAAoB,EAAE,UAAU,CAAC,QAAQ,CAAC,0CAA0C,CACpF,CAAA;AAoDb,eAAO,MAAM,OAAO;;;;;;;;CAQnB,CAAA;AAmCD,eAAO,MAAM,cAAc;;;;;;CAM1B,CAAA"}

package/script/data-pump/metrics.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.metrics = exports.dataPumpPromRegistry = void 0;
+exports.clusterMetrics = exports.metrics = exports.dataPumpPromRegistry = void 0;
 const prom_client_1 = __importStar(require("prom-client"));
 exports.dataPumpPromRegistry = new prom_client_1.default
     .Registry();
@@ -78,3 +78,37 @@ exports.metrics = {
     eventsPulledSizeBytesCounter,
     sdkCommandsCounter,
 };
+// #region Cluster Metrics
+const activeWorkersGauge = new prom_client_1.Gauge({
+    name: "flowcore_data_pump_cluster_active_workers_gauge",
+    help: "The number of active worker connections",
+});
+const leaderStatusGauge = new prom_client_1.Gauge({
+    name: "flowcore_data_pump_cluster_leader_status_gauge",
+    help: "Whether this instance is the leader (1) or not (0)",
+});
+const eventsDistributedCounter = new prom_client_1.Counter({
+    name: "flowcore_data_pump_cluster_events_distributed_counter",
+    help: "The number of events distributed to workers",
+});
+const workerAcksCounter = new prom_client_1.Counter({
+    name: "flowcore_data_pump_cluster_worker_acks_counter",
+    help: "The number of successful worker acknowledgements",
+});
+const workerFailsCounter = new prom_client_1.Counter({
+    name: "flowcore_data_pump_cluster_worker_fails_counter",
+    help: "The number of failed worker deliveries",
+});
+exports.dataPumpPromRegistry.registerMetric(activeWorkersGauge);
+exports.dataPumpPromRegistry.registerMetric(leaderStatusGauge);
+exports.dataPumpPromRegistry.registerMetric(eventsDistributedCounter);
+exports.dataPumpPromRegistry.registerMetric(workerAcksCounter);
+exports.dataPumpPromRegistry.registerMetric(workerFailsCounter);
+exports.clusterMetrics = {
+    activeWorkersGauge,
+    leaderStatusGauge,
+    eventsDistributedCounter,
+    workerAcksCounter,
+    workerFailsCounter,
+};
+// #endregion