@flowcore/data-pump 0.17.0 → 0.18.0

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## [0.18.0](https://github.com/flowcore-io/data-pump/compare/v0.17.0...v0.18.0) (2026-03-26)
+
+
+### Features
+
+* add K8s integration test suite for cluster mode ([9ae9869](https://github.com/flowcore-io/data-pump/commit/9ae98690af3fa7ff4de8ad81fa2f40ba4820274d))
+* add Kubernetes integration test suite for cluster mode ([2012dff](https://github.com/flowcore-io/data-pump/commit/2012dff0ac8acc1e0966d326673d1eb5f961b416))
+* add PulseEmitter for periodic pump status reporting ([3da799b](https://github.com/flowcore-io/data-pump/commit/3da799be0392610768e2b121731d2812efc3c284))
+* unified NATS mode for notification and cluster event distribution ([b616c18](https://github.com/flowcore-io/data-pump/commit/b616c18153680fc7ffe20aa616e37ef366a3b023))
+
+
+### Bug Fixes
+
+* check leader via PG lease table instead of curl ([f2840a0](https://github.com/flowcore-io/data-pump/commit/f2840a021392e2f954533ea56a6188de57b51ef5))
+* delay event delivery to allow cluster formation ([b71f1dc](https://github.com/flowcore-io/data-pump/commit/b71f1dc02eec5f235c239195cf0d766cc5948347))
+* use latest Deno image to support lockfile v5 ([c730f3e](https://github.com/flowcore-io/data-pump/commit/c730f3e4c82d56b9de77331a1f1ecf3c57440096))
+
 ## [0.17.0](https://github.com/flowcore-io/data-pump/compare/v0.16.5...v0.17.0) (2026-03-19)
 
 

package/esm/data-pump/data-pump-cluster.d.ts

@@ -1,13 +1,16 @@
 import * as dntShim from "../_dnt.shims.js";
 import { type FlowcoreDataPumpOptions } from "./data-pump.js";
+import type { FlowcoreDataSource } from "./data-source.js";
 import type { FlowcoreDataPumpCoordinator } from "./types.js";
 export interface FlowcoreDataPumpClusterOptions extends FlowcoreDataPumpOptions {
     coordinator: FlowcoreDataPumpCoordinator;
-    advertisedAddress: string;
+    advertisedAddress?: string;
+    dataSourceOverride?: FlowcoreDataSource;
     leaseTtlMs?: number;
     leaseRenewIntervalMs?: number;
     heartbeatIntervalMs?: number;
     workerConcurrency?: number;
+    clusterKey?: string;
 }
 export declare class FlowcoreDataPumpCluster {
     private readonly options;
@@ -30,7 +33,12 @@ export declare class FlowcoreDataPumpCluster {
     private electionInterval?;
     private workerHandler?;
     private workerFailedHandler?;
+    private natsConnectionManager?;
+    private natsDistLeader?;
+    private natsDistWorker?;
     constructor(options: FlowcoreDataPumpClusterOptions);
+    private get useNatsDistribution();
+    private get natsDistributionSubject();
     get isRunning(): boolean;
     get isLeaderInstance(): boolean;
     get activeWorkerCount(): number;
@@ -68,6 +76,8 @@ export declare class FlowcoreDataPumpCluster {
     private stepDownAsLeader;
     private startPumpAsLeader;
     private distributeEvents;
+    private distributeEventsNats;
+    private distributeEventsWs;
     private selectWorker;
     private startWorkerDiscovery;
     private discoverWorkers;

package/esm/data-pump/data-pump-cluster.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump-cluster.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump-cluster.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAoB,KAAK,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AAE/E,OAAO,KAAK,EAAE,2BAA2B,EAAkB,MAAM,YAAY,CAAA;AAc7E,MAAM,WAAW,8BAA+B,SAAQ,uBAAuB;IAC7E,WAAW,EAAE,2BAA2B,CAAA;IACxC,iBAAiB,EAAE,MAAM,CAAA;IACzB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC3B;AAUD,qBAAa,uBAAuB;IA4BtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IA3BpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;IACzD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAC5C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB;IAExC,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,IAAI,CAAC,CAAkB;IAC/B,OAAO,CAAC,gBAAgB,CAAC,CAAc;IAGvC,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,qBAAqB,CAAI;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAgC;IAC3D,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAG1D,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAC1D,OAAO,CAAC,gBAAgB,CAAC,CAAgC;IAGzD,OAAO,CAAC,aAAa,CAAC,CAA4C;IAClE,OAAO,CAAC,mBAAmB,CAAC,CAAmD;gBAElD,OAAO,EAAE,8BAA8B;IAcpE,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,gBAAgB,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,GAAG,IAAI;IAevC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA2C3B,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,iBAAiB;YAyBX,eAAe;YAgBf,YAAY;IAkB1B,OAAO,CAAC,gBAAgB;IAgCxB,OAAO,CAAC,iBAAiB;YAmBX,gBAAgB;IAqC9B,OAAO,CAAC,YAAY;IAoBpB,OAAO,CAAC,oBAAoB;YAOd,eAAe;IA8B7B,OAAO,CAAC,eAAe;IAqCvB,OAAO,CAAC,sBAAsB;IAiB9B,OAAO,CAAC,iBAAiB;IAiCzB,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,mBAAmB;CA6B5B"}
+{"version":3,"file":"data-pump-cluster.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump-cluster.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAoB,KAAK,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AAC/E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAI1D,OAAO,KAAK,EAAE,2BAA2B,EAAkB,MAAM,YAAY,CAAA;AAc7E,MAAM,WAAW,8BAA+B,SAAQ,uBAAuB;IAC7E,WAAW,EAAE,2BAA2B,CAAA;IACxC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,kBAAkB,CAAC,EAAE,kBAAkB,CAAA;IACvC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAUD,qBAAa,uBAAuB;IAiCtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAhCpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;IACzD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAC5C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB;IAExC,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,IAAI,CAAC,CAAkB;IAC/B,OAAO,CAAC,gBAAgB,CAAC,CAAc;IAGvC,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,qBAAqB,CAAI;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAgC;IAC3D,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAG1D,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAC1D,OAAO,CAAC,gBAAgB,CAAC,CAAgC;IAGzD,OAAO,CAAC,aAAa,CAAC,CAA4C;IAClE,OAAO,CAAC,mBAAmB,CAAC,CAAmD;IAG/E,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IACrD,OAAO,CAAC,cAAc,CAAC,CAAwB;IAC/C,OAAO,CAAC,cAAc,CAAC,CAAwB;gBAElB,OAAO,EAAE,8BAA8B;IAmBpE,OAAO,KAAK,mBAAmB,GAE9B;IAED,OAAO,KAAK,uBAAuB,GAIlC;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,gBAAgB,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,GAAG,IAAI;IAevC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuCtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD3B,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,iBAAiB;YAyBX,eAAe;YAgBf,YAAY;IA0B1B,OAAO,CAAC,gBAAgB;IAmCxB,OAAO,CAAC,iBAAiB;IAmBzB,OAAO,CAAC,gBAAgB;YAOV,oBAAoB;YA2BpB,kBAAkB;IAqChC,OAAO,CAAC,YAAY;IAoBpB,OAAO,CAAC,oBAAoB;YAOd,eAAe;IA8B7B,OAAO,CAAC,eAAe;IAqCvB,OAAO,CAAC,sBAAsB;IAiB9B,OAAO,CAAC,iBAAiB;IAiCzB,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,mBAAmB;CA6B5B"}

package/esm/data-pump/data-pump-cluster.js

@@ -1,6 +1,8 @@
 import * as dntShim from "../_dnt.shims.js";
 import { FlowcoreDataPump } from "./data-pump.js";
 import { clusterMetrics } from "./metrics.js";
+import { NatsConnectionManager } from "./nats-connection.js";
+import { NatsDistributionLeader, NatsDistributionWorker } from "./nats-distribution.js";
 import { DeliveryTracker, WsConnection } from "./ws-protocol.js";
 const LEASE_KEY = "flowcore-data-pump-leader";
 const DISCOVERY_POLL_INTERVAL_MS = 10_000;
@@ -137,6 +139,29 @@ export class FlowcoreDataPumpCluster {
             writable: true,
             value: void 0
         });
+        // NATS distribution state
+        Object.defineProperty(this, "natsConnectionManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "natsDistLeader", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "natsDistWorker", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // validate: WS mode requires advertisedAddress
+        if (!this.useNatsDistribution && !options.advertisedAddress) {
+            throw new Error("advertisedAddress is required when not using NATS distribution (notifier.type !== 'nats')");
+        }
         this.instanceId = crypto.randomUUID();
         this.coordinator = options.coordinator;
         this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
@@ -148,6 +173,14 @@ export class FlowcoreDataPumpCluster {
         this.workerHandler = options.processor?.handler;
         this.workerFailedHandler = options.processor?.failedHandler;
     }
+    get useNatsDistribution() {
+        return this.options.notifier?.type === "nats";
+    }
+    get natsDistributionSubject() {
+        const key = this.options.clusterKey ??
+            `${this.options.dataSource.tenant}.${this.options.dataSource.dataCore}.${this.options.dataSource.flowType}`;
+        return `data-pump.distribute.${key}`;
+    }
     get isRunning() {
         return this.running;
     }
@@ -201,11 +234,26 @@ export class FlowcoreDataPumpCluster {
         }
         this.running = true;
         this.logger?.info("Starting cluster instance", { instanceId: this.instanceId });
-        // Step 1: Register in DB
-        await this.coordinator.register(this.instanceId, this.options.advertisedAddress);
-        // Step 2: Start heartbeat
+        if (this.useNatsDistribution) {
+            // NATS mode: connect shared NATS, start worker subscription
+            const natsServers = this.options.notifier?.type === "nats" ? this.options.notifier.servers : [];
+            this.natsConnectionManager = new NatsConnectionManager(natsServers, this.logger);
+            const conn = await this.natsConnectionManager.connect();
+            // all instances subscribe as workers (leader also processes if no workers pick up)
+            if (this.workerHandler) {
+                this.natsDistWorker = new NatsDistributionWorker(conn, this.natsDistributionSubject, this.workerHandler, this.logger);
+                this.natsDistWorker.start();
+            }
+            // register with placeholder address (not used for routing in NATS mode)
+            await this.coordinator.register(this.instanceId, "nats://distributed");
+        }
+        else {
+            // WS mode: register with actual address
+            await this.coordinator.register(this.instanceId, this.options.advertisedAddress);
+        }
+        // Start heartbeat
         this.startHeartbeat();
-        // Step 3: Start leader election loop
+        // Start leader election loop
         this.startElectionLoop();
     }
     async stop() {
@@ -227,6 +275,15 @@ export class FlowcoreDataPumpCluster {
         // close worker connection to leader
         this.leaderConnection?.close();
         this.leaderConnection = undefined;
+        // stop NATS distribution
+        this.natsDistWorker?.stop();
+        this.natsDistWorker = undefined;
+        this.natsDistLeader = undefined;
+        // close shared NATS connection
+        if (this.natsConnectionManager) {
+            await this.natsConnectionManager.close();
+            this.natsConnectionManager = undefined;
+        }
         // unregister
         try {
             await this.coordinator.releaseLease(this.instanceId, LEASE_KEY);
@@ -304,10 +361,16 @@ export class FlowcoreDataPumpCluster {
         // close any incoming leader connection (we are leader now)
         this.leaderConnection?.close();
         this.leaderConnection = undefined;
-        // start worker discovery
-        this.startWorkerDiscovery();
-        // initial discovery
-        await this.discoverWorkers();
+        if (this.useNatsDistribution) {
+            // NATS mode: create leader distributor, no WS discovery needed
+            const conn = await this.natsConnectionManager.connect();
+            this.natsDistLeader = new NatsDistributionLeader(conn, this.natsDistributionSubject, this.logger);
+        }
+        else {
+            // WS mode: start worker discovery
+            this.startWorkerDiscovery();
+            await this.discoverWorkers();
+        }
         // start the pump with distribution processor
         this.startPumpAsLeader();
     }
@@ -326,7 +389,9 @@ export class FlowcoreDataPumpCluster {
             clearInterval(this.leaseRenewInterval);
             this.leaseRenewInterval = undefined;
         }
-        // disconnect all workers
+        // NATS mode: close leader distributor (worker subscription stays)
+        this.natsDistLeader = undefined;
+        // disconnect all WS workers
         for (const worker of this.workers.values()) {
             worker.deliveryTracker.rejectAll(new Error("Leader stepping down"));
             worker.connection.close();
@@ -351,12 +416,43 @@ export class FlowcoreDataPumpCluster {
                 failedHandler: this.workerFailedHandler,
             },
         };
-        this.pump = FlowcoreDataPump.create(pumpOptions);
+        this.pump = FlowcoreDataPump.create(pumpOptions, this.options.dataSourceOverride);
         this.pump.start().catch((error) => {
             this.logger?.error("Pump error in leader mode", { error });
         });
     }
-    async distributeEvents(events) {
+    distributeEvents(events) {
+        if (this.useNatsDistribution) {
+            return this.distributeEventsNats(events);
+        }
+        return this.distributeEventsWs(events);
+    }
+    async distributeEventsNats(events) {
+        if (!this.natsDistLeader) {
+            // no NATS leader distributor - process locally as fallback
+            this.logger?.debug("No NATS leader distributor, processing locally");
+            if (this.workerHandler) {
+                await this.workerHandler(events);
+            }
+            return;
+        }
+        clusterMetrics.eventsDistributedCounter.inc(events.length);
+        try {
+            await this.natsDistLeader.distribute(events);
+            clusterMetrics.workerAcksCounter.inc();
+        }
+        catch (error) {
+            clusterMetrics.workerFailsCounter.inc();
+            // fallback to local processing on NATS failure
+            this.logger?.warn("NATS distribution failed, processing locally", {
+                error: error instanceof Error ? error.message : "unknown",
+            });
+            if (this.workerHandler) {
+                await this.workerHandler(events);
+            }
+        }
+    }
+    async distributeEventsWs(events) {
         const worker = this.selectWorker();
         if (!worker) {
             // no workers available - process locally (single-instance fallback)

package/esm/data-pump/data-pump.d.ts

@@ -1,5 +1,6 @@
-import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
+import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import { FlowcoreDataSource } from "./data-source.js";
+import { type PulseSnapshot } from "./pulse.js";
 import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpProcessor, FlowcoreDataPumpState, FlowcoreDataPumpStateManager, FlowcoreLogger } from "./types.js";
 interface FlowcoreDataPumpNotifierNatsOptions {
     type: "nats";
@@ -31,6 +32,11 @@ export interface FlowcoreDataPumpOptions {
     baseUrlOverride?: string;
     noTranslation?: boolean;
     directMode?: boolean;
+    pulse?: {
+        url: string;
+        intervalMs?: number;
+        pathwayId: string;
+    };
 }
 export declare class FlowcoreDataPump {
     readonly dataSource: FlowcoreDataSource;
@@ -47,8 +53,14 @@ export declare class FlowcoreDataPump {
     private stopAtState?;
     private isLive;
     private finallyFailedHandler?;
+    private pulseEmitter?;
+    private startedAt;
+    private acknowledgedCount;
+    private failedCount;
+    private pulledCount;
     private constructor();
     get isRunning(): boolean;
+    getSnapshot(): PulseSnapshot | null;
     static create(options: FlowcoreDataPumpOptions, dataSourceOverride?: FlowcoreDataSource): FlowcoreDataPump;
     start(callback?: (error?: Error) => void): Promise<void>;
     restart(state: FlowcoreDataPumpState, stopAt?: Date | null): void;

package/esm/data-pump/data-pump.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAGrD,OAAO,KAAK,EACR,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,cAAc,EACjB,MAAM,YAAY,CAAA;AAEnB,UAAU,mCAAmC;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED,UAAU,wCAAwC;IAChD,IAAI,EAAE,WAAW,CAAA;CAClB;AAED,UAAU,qCAAqC;IAC7C,IAAI,EAAE,QAAQ,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,KAAK,+BAA+B,GAChC,mCAAmC,GACnC,wCAAwC,GACxC,qCAAqC,CAAA;AAEzC,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;IACtC,YAAY,EAAE,4BAA4B,CAAA;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAC9B,SAAS,CAAC,EAAE,yBAAyB,CAAA;IACrC,QAAQ,CAAC,EAAE,+BAA+B,CAAA;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAmBD,qBAAa,gBAAgB;aAYT,UAAU,EAAE,kBAAkB;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAf1B,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAC,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,oBAAoB,CAAC,CAAmD;IAEhF,OAAO;IAaP,IAAW,SAAS,IAAI,OAAO,CAE9B;WAEa,MAAM,CAAC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,gBAAgB;IA0CpG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA2C9D,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAQjE,IAAI,CAAC,SAAS,UAAQ,GAAG,IAAI;IAWpC,OAAO,CAAC,WAAW;YAaL,IAAI;IAmFL,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAqCvD,cAAc,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI;IAInE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE;IA0B9B,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;YA4BtB,MAAM;YAkDN,WAAW;IAiBzB,OAAO,CAAC,mBAAmB;IA4D3B,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,YAAY,CAAC,CAAY;YACnB,aAAa;IAO3B,OAAO,CAAC,qBAAqB,CAAC,CAAY;YAC5B,sBAAsB;IAOpC,OAAO,CAAC,iBAAiB,CAAC,CAAY;YACxB,kBAAkB;CAWjC"}
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAGrD,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,YAAY,CAAA;AAC7D,OAAO,KAAK,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,cAAc,EACf,MAAM,YAAY,CAAA;AAEnB,UAAU,mCAAmC;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED,UAAU,wCAAwC;IAChD,IAAI,EAAE,WAAW,CAAA;CAClB;AAED,UAAU,qCAAqC;IAC7C,IAAI,EAAE,QAAQ,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,KAAK,+BAA+B,GAChC,mCAAmC,GACnC,wCAAwC,GACxC,qCAAqC,CAAA;AAEzC,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;IACtC,YAAY,EAAE,4BAA4B,CAAA;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAC9B,SAAS,CAAC,EAAE,yBAAyB,CAAA;IACrC,QAAQ,CAAC,EAAE,+BAA+B,CAAA;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,KAAK,CAAC,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,SAAS,EAAE,MAAM,CAAA;KAClB,CAAA;CACF;AAmBD,qBAAa,gBAAgB;aAiBT,UAAU,EAAE,kBAAkB;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IApB1B,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAC,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,oBAAoB,CAAC,CAAmD;IAChF,OAAO,CAAC,YAAY,CAAC,CAAc;IACnC,OAAO,CAAC,SAAS,CAAI;IACrB,OAAO,CAAC,iBAAiB,CAAI;IAC7B,OAAO,CAAC,WAAW,CAAI;IACvB,OAAO,CAAC,WAAW,CAAI;IAEvB,OAAO;IAaP,IAAW,SAAS,IAAI,OAAO,CAE9B;IAEM,WAAW,IAAI,aAAa,GAAG,IAAI;WAoB5B,MAAM,CAAC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,gBAAgB;IA6DpG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA6C9D,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAQjE,IAAI,CAAC,SAAS,UAAQ,GAAG,IAAI;IAYpC,OAAO,CAAC,WAAW;YAaL,IAAI;IAoFL,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAqCvD,cAAc,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI;IAInE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE;IA2B9B,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;YA6BtB,MAAM;YAkDN,WAAW;IAiBzB,OAAO,CAAC,mBAAmB;IA4D3B,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,YAAY,CAAC,CAAY;YACnB,aAAa;IAO3B,OAAO,CAAC,qBAAqB,CAAC,CAAY;YAC5B,sBAAsB;IAOpC,OAAO,CAAC,iBAAiB,CAAC,CAAY;YACxB,kBAAkB;CAWjC"}

package/esm/data-pump/data-pump.js

@@ -4,6 +4,7 @@ import { format, startOfHour } from "date-fns";
 import { FlowcoreDataSource } from "./data-source.js";
 import { metrics } from "./metrics.js";
 import { FlowcoreNotifier } from "./notifier.js";
+import { PulseEmitter } from "./pulse.js";
 export class FlowcoreDataPump {
     constructor(dataSource, notifier, stateManager, options, logger) {
         Object.defineProperty(this, "dataSource", {
@@ -90,6 +91,36 @@ export class FlowcoreDataPump {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "pulseEmitter", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "startedAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "acknowledgedCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "failedCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "pulledCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
         // #endregion
         // #region Waiters
         Object.defineProperty(this, "waiterEvents", {
@@ -118,6 +149,26 @@ export class FlowcoreDataPump {
     get isRunning() {
         return this.running;
     }
+    getSnapshot() {
+        if (!this.running)
+            return null;
+        const reserved = this.buffer.filter((b) => b.status === "reserved").length;
+        const sizeBytes = this.buffer.reduce((sum, b) => sum + JSON.stringify(b.event.payload).length, 0);
+        return {
+            pathwayId: this.pulseEmitter ? "" : "", // set by caller
+            flowType: this.dataSource.flowType,
+            timeBucket: this.bufferState.timeBucket,
+            eventId: this.bufferState.eventId,
+            isLive: this.isLive,
+            bufferDepth: this.buffer.length,
+            bufferReserved: reserved,
+            bufferSizeBytes: sizeBytes,
+            acknowledgedTotal: this.acknowledgedCount,
+            failedTotal: this.failedCount,
+            pulledTotal: this.pulledCount,
+            uptimeMs: this.startedAt ? Date.now() - this.startedAt : 0,
+        };
+    }
     static create(options, dataSourceOverride) {
         if ("apiKey" in options.auth && !options.auth.apiKeyId) {
             const parts = options.auth.apiKey.split("_");
@@ -142,7 +193,7 @@ export class FlowcoreDataPump {
             directMode: options.directMode,
             noTranslation: options.noTranslation,
         });
-        return new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
+        const pump = new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
             bufferSize: options.bufferSize ?? 1000,
             bufferThreshold: Math.ceil((options.bufferSize ?? 1000) * 0.1),
             maxRedeliveryCount: options.maxRedeliveryCount ?? 3,
@@ -151,6 +202,21 @@ export class FlowcoreDataPump {
             processor: options.processor,
             stopAt: options.stopAt,
         }, options.logger);
+        if (options.pulse) {
+            const pathwayId = options.pulse.pathwayId;
+            pump.pulseEmitter = new PulseEmitter({
+                url: options.pulse.url,
+                intervalMs: options.pulse.intervalMs,
+                auth: options.auth,
+                logger: options.logger,
+            }, () => {
+                const snapshot = pump.getSnapshot();
+                if (snapshot)
+                    snapshot.pathwayId = pathwayId;
+                return snapshot;
+            });
+        }
+        return pump;
     }
     async start(callback) {
         this.isLive = false;
@@ -158,8 +224,10 @@ export class FlowcoreDataPump {
             throw new Error("Data pump already running");
         }
         this.running = true;
+        this.startedAt = Date.now();
         this.nextCursor = undefined;
         this.updateMetricsGauges();
+        this.pulseEmitter?.start();
         const currentState = await this.stateManager.getState();
         const timeBucket = currentState
             ? await this.dataSource.getClosestTimeBucket(currentState.timeBucket)
@@ -198,6 +266,7 @@ export class FlowcoreDataPump {
         this.running = false;
         this.buffer = [];
         this.updateMetricsGauges();
+        this.pulseEmitter?.stop();
         this.abortController?.abort();
         this.waiterBufferThreshold?.();
         if (!isRestart) {
@@ -230,6 +299,7 @@ export class FlowcoreDataPump {
                 break;
             }
             this.logger?.debug(`fetched ${events.length} events`);
+            this.pulledCount += events.length;
             this.buffer.push(...events.map((event) => ({ event, status: "open", deliveryCount: 0 })));
             this.nextCursor = nextCursor;
             this.updateMetricsGauges();
@@ -316,6 +386,7 @@ export class FlowcoreDataPump {
         this.buffer = this.buffer.filter((event) => {
             if (eventIds.includes(event.event.eventId)) {
                 this.incMetricsCounter("acknowledged", event.event.eventType, 1);
+                this.acknowledgedCount++;
                 return false;
             }
             return true;
@@ -338,6 +409,7 @@ export class FlowcoreDataPump {
         this.buffer = this.buffer.filter((event) => {
             if (eventIds.includes(event.event.eventId)) {
                 this.incMetricsCounter("failed", event.event.eventType, 1);
+                this.failedCount++;
                 failedEvents.push(event.event);
                 return false;
             }

package/esm/data-pump/data-source.d.ts

@@ -2,7 +2,7 @@
  * @module data-source
  * @description Provides functionality to interact with Flowcore data sources
  */
-import { type EventListOutput, type FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
+import { type EventListOutput, type FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpState } from "./types.js";
 /**
  * Options for configuring a Flowcore data source

package/esm/data-pump/data-source.js

@@ -2,7 +2,7 @@
  * @module data-source
  * @description Provides functionality to interact with Flowcore data sources
  */
-import { DataCoreFetchCommand, EventListCommand, EventTypeListCommand, FlowTypeFetchCommand, TenantTranslateNameToIdCommand, TimeBucketListCommand, } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
+import { DataCoreFetchCommand, EventListCommand, EventTypeListCommand, FlowTypeFetchCommand, TenantTranslateNameToIdCommand, TimeBucketListCommand, } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import { getFlowcoreClient } from "./flowcore-client.js";
 /**
  * Provides methods to interact with a Flowcore data source

package/esm/data-pump/flowcore-client.d.ts

@@ -1,4 +1,4 @@
-import { type Command, FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
+import { type Command, FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import type { FlowcoreDataPumpAuth } from "./types.js";
 declare class FlowcoreClientWithMetrics extends FlowcoreClient {
     execute<Input, Output>(command: Command<Input, Output>, directMode?: boolean): Promise<Output>;

package/esm/data-pump/flowcore-client.js

@@ -1,4 +1,4 @@
-import { FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.71.0/src/mod.js";
+import { FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import { metrics } from "./metrics.js";
 class FlowcoreClientWithMetrics extends FlowcoreClient {
     execute(command, directMode) {

package/esm/data-pump/nats-connection.d.ts

@@ -0,0 +1,13 @@
+import * as Nats from "nats";
+import type { FlowcoreLogger } from "./types.js";
+export declare class NatsConnectionManager {
+    private readonly servers;
+    private readonly logger?;
+    private connection?;
+    private connecting?;
+    constructor(servers: string[], logger?: FlowcoreLogger | undefined);
+    connect(): Promise<Nats.NatsConnection>;
+    close(): Promise<void>;
+    get isConnected(): boolean;
+}
+//# sourceMappingURL=nats-connection.d.ts.map

package/esm/data-pump/nats-connection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nats-connection.d.ts","sourceRoot":"","sources":["../../src/data-pump/nats-connection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,qBAAa,qBAAqB;IAK9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAL1B,OAAO,CAAC,UAAU,CAAC,CAAqB;IACxC,OAAO,CAAC,UAAU,CAAC,CAA8B;gBAG9B,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,CAAC,EAAE,cAAc,YAAA;IAG1C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC;IAsBjC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B,IAAI,WAAW,IAAI,OAAO,CAEzB;CACF"}

package/esm/data-pump/nats-connection.js

@@ -0,0 +1,58 @@
+import * as Nats from "nats";
+export class NatsConnectionManager {
+    constructor(servers, logger) {
+        Object.defineProperty(this, "servers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: servers
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: logger
+        });
+        Object.defineProperty(this, "connection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "connecting", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+    }
+    connect() {
+        if (this.connection && !this.connection.isClosed()) {
+            return Promise.resolve(this.connection);
+        }
+        // deduplicate concurrent connect calls
+        if (this.connecting) {
+            return this.connecting;
+        }
+        this.connecting = (async () => {
+            this.logger?.debug("Connecting to NATS", { servers: this.servers });
+            const conn = await Nats.connect({ servers: this.servers });
+            this.connection = conn;
+            this.connecting = undefined;
+            this.logger?.info("Connected to NATS");
+            return conn;
+        })();
+        return this.connecting;
+    }
+    async close() {
+        if (this.connection && !this.connection.isClosed()) {
+            this.logger?.debug("Closing NATS connection");
+            await this.connection.drain();
+            this.connection = undefined;
+        }
+        this.connecting = undefined;
+    }
+    get isConnected() {
+        return !!this.connection && !this.connection.isClosed();
+    }
+}

package/esm/data-pump/nats-distribution.d.ts

@@ -0,0 +1,34 @@
+import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
+import * as Nats from "nats";
+import type { FlowcoreLogger } from "./types.js";
+export interface NatsDistributionRequest {
+    deliveryId: string;
+    events: FlowcoreEvent[];
+}
+export interface NatsDistributionReply {
+    status: "ack" | "fail";
+    error?: string;
+}
+export declare class NatsDistributionLeader {
+    private readonly connection;
+    private readonly subject;
+    private readonly logger?;
+    private readonly timeoutMs;
+    private readonly codec;
+    constructor(connection: Nats.NatsConnection, subject: string, logger?: FlowcoreLogger | undefined, timeoutMs?: number);
+    distribute(events: FlowcoreEvent[]): Promise<void>;
+}
+export declare class NatsDistributionWorker {
+    private readonly connection;
+    private readonly subject;
+    private readonly handler;
+    private readonly logger?;
+    private subscription?;
+    private readonly codec;
+    private readonly queueGroup;
+    constructor(connection: Nats.NatsConnection, subject: string, handler: (events: FlowcoreEvent[]) => Promise<void>, logger?: FlowcoreLogger | undefined);
+    start(): void;
+    stop(): void;
+    private handleMessage;
+}
+//# sourceMappingURL=nats-distribution.d.ts.map

package/esm/data-pump/nats-distribution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nats-distribution.d.ts","sourceRoot":"","sources":["../../src/data-pump/nats-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AACnF,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAIhD,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,aAAa,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,KAAK,GAAG,MAAM,CAAA;IACtB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,qBAAa,sBAAsB;IAI/B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAN5B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoE;gBAGvE,UAAU,EAAE,IAAI,CAAC,cAAc,EAC/B,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,cAAc,YAAA,EACvB,SAAS,GAAE,MAAmC;IAG3D,UAAU,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAiBzD;AAED,qBAAa,sBAAsB;IAM/B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAR1B,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoE;IAC1F,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;gBAG9B,UAAU,EAAE,IAAI,CAAC,cAAc,EAC/B,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,EACnD,MAAM,CAAC,EAAE,cAAc,YAAA;IAG1C,KAAK,IAAI,IAAI;IAWb,IAAI,IAAI,IAAI;YAQE,aAAa;CAoB5B"}