@flowcore/data-pump 0.16.5 → 0.18.0

package/esm/data-pump/data-pump.js

@@ -4,6 +4,7 @@ import { format, startOfHour } from "date-fns";
 import { FlowcoreDataSource } from "./data-source.js";
 import { metrics } from "./metrics.js";
 import { FlowcoreNotifier } from "./notifier.js";
+import { PulseEmitter } from "./pulse.js";
 export class FlowcoreDataPump {
     constructor(dataSource, notifier, stateManager, options, logger) {
         Object.defineProperty(this, "dataSource", {
@@ -90,6 +91,36 @@ export class FlowcoreDataPump {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "pulseEmitter", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "startedAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "acknowledgedCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "failedCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "pulledCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
         // #endregion
         // #region Waiters
         Object.defineProperty(this, "waiterEvents", {
@@ -118,6 +149,26 @@ export class FlowcoreDataPump {
     get isRunning() {
         return this.running;
     }
+    getSnapshot() {
+        if (!this.running)
+            return null;
+        const reserved = this.buffer.filter((b) => b.status === "reserved").length;
+        const sizeBytes = this.buffer.reduce((sum, b) => sum + JSON.stringify(b.event.payload).length, 0);
+        return {
+            pathwayId: this.pulseEmitter ? "" : "", // set by caller
+            flowType: this.dataSource.flowType,
+            timeBucket: this.bufferState.timeBucket,
+            eventId: this.bufferState.eventId,
+            isLive: this.isLive,
+            bufferDepth: this.buffer.length,
+            bufferReserved: reserved,
+            bufferSizeBytes: sizeBytes,
+            acknowledgedTotal: this.acknowledgedCount,
+            failedTotal: this.failedCount,
+            pulledTotal: this.pulledCount,
+            uptimeMs: this.startedAt ? Date.now() - this.startedAt : 0,
+        };
+    }
     static create(options, dataSourceOverride) {
         if ("apiKey" in options.auth && !options.auth.apiKeyId) {
             const parts = options.auth.apiKey.split("_");
@@ -142,7 +193,7 @@ export class FlowcoreDataPump {
             directMode: options.directMode,
             noTranslation: options.noTranslation,
         });
-        return new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
+        const pump = new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
             bufferSize: options.bufferSize ?? 1000,
             bufferThreshold: Math.ceil((options.bufferSize ?? 1000) * 0.1),
             maxRedeliveryCount: options.maxRedeliveryCount ?? 3,
@@ -151,6 +202,21 @@ export class FlowcoreDataPump {
             processor: options.processor,
             stopAt: options.stopAt,
         }, options.logger);
+        if (options.pulse) {
+            const pathwayId = options.pulse.pathwayId;
+            pump.pulseEmitter = new PulseEmitter({
+                url: options.pulse.url,
+                intervalMs: options.pulse.intervalMs,
+                auth: options.auth,
+                logger: options.logger,
+            }, () => {
+                const snapshot = pump.getSnapshot();
+                if (snapshot)
+                    snapshot.pathwayId = pathwayId;
+                return snapshot;
+            });
+        }
+        return pump;
     }
     async start(callback) {
         this.isLive = false;
@@ -158,8 +224,10 @@ export class FlowcoreDataPump {
             throw new Error("Data pump already running");
         }
         this.running = true;
+        this.startedAt = Date.now();
         this.nextCursor = undefined;
         this.updateMetricsGauges();
+        this.pulseEmitter?.start();
         const currentState = await this.stateManager.getState();
         const timeBucket = currentState
             ? await this.dataSource.getClosestTimeBucket(currentState.timeBucket)
@@ -198,6 +266,7 @@ export class FlowcoreDataPump {
         this.running = false;
         this.buffer = [];
         this.updateMetricsGauges();
+        this.pulseEmitter?.stop();
         this.abortController?.abort();
         this.waiterBufferThreshold?.();
         if (!isRestart) {
@@ -230,6 +299,7 @@ export class FlowcoreDataPump {
                 break;
             }
             this.logger?.debug(`fetched ${events.length} events`);
+            this.pulledCount += events.length;
             this.buffer.push(...events.map((event) => ({ event, status: "open", deliveryCount: 0 })));
             this.nextCursor = nextCursor;
             this.updateMetricsGauges();
@@ -316,6 +386,7 @@ export class FlowcoreDataPump {
         this.buffer = this.buffer.filter((event) => {
             if (eventIds.includes(event.event.eventId)) {
                 this.incMetricsCounter("acknowledged", event.event.eventType, 1);
+                this.acknowledgedCount++;
                 return false;
             }
             return true;
@@ -338,6 +409,7 @@ export class FlowcoreDataPump {
         this.buffer = this.buffer.filter((event) => {
             if (eventIds.includes(event.event.eventId)) {
                 this.incMetricsCounter("failed", event.event.eventType, 1);
+                this.failedCount++;
                 failedEvents.push(event.event);
                 return false;
             }

package/esm/data-pump/data-source.d.ts

@@ -2,7 +2,7 @@
  * @module data-source
  * @description Provides functionality to interact with Flowcore data sources
  */
-import { type EventListOutput, type FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import { type EventListOutput, type FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpState } from "./types.js";
 /**
  * Options for configuring a Flowcore data source

package/esm/data-pump/data-source.js

@@ -2,7 +2,7 @@
  * @module data-source
  * @description Provides functionality to interact with Flowcore data sources
  */
-import { DataCoreFetchCommand, EventListCommand, EventTypeListCommand, FlowTypeFetchCommand, TenantTranslateNameToIdCommand, TimeBucketListCommand, } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import { DataCoreFetchCommand, EventListCommand, EventTypeListCommand, FlowTypeFetchCommand, TenantTranslateNameToIdCommand, TimeBucketListCommand, } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import { getFlowcoreClient } from "./flowcore-client.js";
 /**
  * Provides methods to interact with a Flowcore data source

package/esm/data-pump/flowcore-client.d.ts

@@ -1,4 +1,4 @@
-import { type Command, FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import { type Command, FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import type { FlowcoreDataPumpAuth } from "./types.js";
 declare class FlowcoreClientWithMetrics extends FlowcoreClient {
     execute<Input, Output>(command: Command<Input, Output>, directMode?: boolean): Promise<Output>;

package/esm/data-pump/flowcore-client.js

@@ -1,4 +1,4 @@
-import { FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
+import { FlowcoreClient } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import { metrics } from "./metrics.js";
 class FlowcoreClientWithMetrics extends FlowcoreClient {
     execute(command, directMode) {

package/esm/data-pump/metrics.d.ts

@@ -9,4 +9,11 @@ export declare const metrics: {
     eventsPulledSizeBytesCounter: PromClient.Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
     sdkCommandsCounter: PromClient.Counter<"command">;
 };
+export declare const clusterMetrics: {
+    activeWorkersGauge: PromClient.Gauge<string>;
+    leaderStatusGauge: PromClient.Gauge<string>;
+    eventsDistributedCounter: PromClient.Counter<string>;
+    workerAcksCounter: PromClient.Counter<string>;
+    workerFailsCounter: PromClient.Counter<string>;
+};
 //# sourceMappingURL=metrics.d.ts.map

package/esm/data-pump/metrics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/data-pump/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,UAA8B,MAAM,aAAa,CAAA;AAExD,eAAO,MAAM,oBAAoB,EAAE,UAAU,CAAC,QAAQ,CAAC,0CAA0C,CACpF,CAAA;AAoDb,eAAO,MAAM,OAAO;;;;;;;;CAQnB,CAAA"}
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/data-pump/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,UAA8B,MAAM,aAAa,CAAA;AAExD,eAAO,MAAM,oBAAoB,EAAE,UAAU,CAAC,QAAQ,CAAC,0CAA0C,CACpF,CAAA;AAoDb,eAAO,MAAM,OAAO;;;;;;;;CAQnB,CAAA;AAmCD,eAAO,MAAM,cAAc;;;;;;CAM1B,CAAA"}

package/esm/data-pump/metrics.js

@@ -52,3 +52,37 @@ export const metrics = {
     eventsPulledSizeBytesCounter,
     sdkCommandsCounter,
 };
+// #region Cluster Metrics
+const activeWorkersGauge = new Gauge({
+    name: "flowcore_data_pump_cluster_active_workers_gauge",
+    help: "The number of active worker connections",
+});
+const leaderStatusGauge = new Gauge({
+    name: "flowcore_data_pump_cluster_leader_status_gauge",
+    help: "Whether this instance is the leader (1) or not (0)",
+});
+const eventsDistributedCounter = new Counter({
+    name: "flowcore_data_pump_cluster_events_distributed_counter",
+    help: "The number of events distributed to workers",
+});
+const workerAcksCounter = new Counter({
+    name: "flowcore_data_pump_cluster_worker_acks_counter",
+    help: "The number of successful worker acknowledgements",
+});
+const workerFailsCounter = new Counter({
+    name: "flowcore_data_pump_cluster_worker_fails_counter",
+    help: "The number of failed worker deliveries",
+});
+dataPumpPromRegistry.registerMetric(activeWorkersGauge);
+dataPumpPromRegistry.registerMetric(leaderStatusGauge);
+dataPumpPromRegistry.registerMetric(eventsDistributedCounter);
+dataPumpPromRegistry.registerMetric(workerAcksCounter);
+dataPumpPromRegistry.registerMetric(workerFailsCounter);
+export const clusterMetrics = {
+    activeWorkersGauge,
+    leaderStatusGauge,
+    eventsDistributedCounter,
+    workerAcksCounter,
+    workerFailsCounter,
+};
+// #endregion

package/esm/data-pump/nats-connection.d.ts

@@ -0,0 +1,13 @@
+import * as Nats from "nats";
+import type { FlowcoreLogger } from "./types.js";
+export declare class NatsConnectionManager {
+    private readonly servers;
+    private readonly logger?;
+    private connection?;
+    private connecting?;
+    constructor(servers: string[], logger?: FlowcoreLogger | undefined);
+    connect(): Promise<Nats.NatsConnection>;
+    close(): Promise<void>;
+    get isConnected(): boolean;
+}
+//# sourceMappingURL=nats-connection.d.ts.map

package/esm/data-pump/nats-connection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nats-connection.d.ts","sourceRoot":"","sources":["../../src/data-pump/nats-connection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,qBAAa,qBAAqB;IAK9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAL1B,OAAO,CAAC,UAAU,CAAC,CAAqB;IACxC,OAAO,CAAC,UAAU,CAAC,CAA8B;gBAG9B,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,CAAC,EAAE,cAAc,YAAA;IAG1C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC;IAsBjC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B,IAAI,WAAW,IAAI,OAAO,CAEzB;CACF"}

package/esm/data-pump/nats-connection.js

@@ -0,0 +1,58 @@
+import * as Nats from "nats";
+export class NatsConnectionManager {
+    constructor(servers, logger) {
+        Object.defineProperty(this, "servers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: servers
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: logger
+        });
+        Object.defineProperty(this, "connection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "connecting", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+    }
+    connect() {
+        if (this.connection && !this.connection.isClosed()) {
+            return Promise.resolve(this.connection);
+        }
+        // deduplicate concurrent connect calls
+        if (this.connecting) {
+            return this.connecting;
+        }
+        this.connecting = (async () => {
+            this.logger?.debug("Connecting to NATS", { servers: this.servers });
+            const conn = await Nats.connect({ servers: this.servers });
+            this.connection = conn;
+            this.connecting = undefined;
+            this.logger?.info("Connected to NATS");
+            return conn;
+        })();
+        return this.connecting;
+    }
+    async close() {
+        if (this.connection && !this.connection.isClosed()) {
+            this.logger?.debug("Closing NATS connection");
+            await this.connection.drain();
+            this.connection = undefined;
+        }
+        this.connecting = undefined;
+    }
+    get isConnected() {
+        return !!this.connection && !this.connection.isClosed();
+    }
+}

package/esm/data-pump/nats-distribution.d.ts

@@ -0,0 +1,34 @@
+import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
+import * as Nats from "nats";
+import type { FlowcoreLogger } from "./types.js";
+export interface NatsDistributionRequest {
+    deliveryId: string;
+    events: FlowcoreEvent[];
+}
+export interface NatsDistributionReply {
+    status: "ack" | "fail";
+    error?: string;
+}
+export declare class NatsDistributionLeader {
+    private readonly connection;
+    private readonly subject;
+    private readonly logger?;
+    private readonly timeoutMs;
+    private readonly codec;
+    constructor(connection: Nats.NatsConnection, subject: string, logger?: FlowcoreLogger | undefined, timeoutMs?: number);
+    distribute(events: FlowcoreEvent[]): Promise<void>;
+}
+export declare class NatsDistributionWorker {
+    private readonly connection;
+    private readonly subject;
+    private readonly handler;
+    private readonly logger?;
+    private subscription?;
+    private readonly codec;
+    private readonly queueGroup;
+    constructor(connection: Nats.NatsConnection, subject: string, handler: (events: FlowcoreEvent[]) => Promise<void>, logger?: FlowcoreLogger | undefined);
+    start(): void;
+    stop(): void;
+    private handleMessage;
+}
+//# sourceMappingURL=nats-distribution.d.ts.map

package/esm/data-pump/nats-distribution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nats-distribution.d.ts","sourceRoot":"","sources":["../../src/data-pump/nats-distribution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AACnF,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAIhD,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,aAAa,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,KAAK,GAAG,MAAM,CAAA;IACtB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,qBAAa,sBAAsB;IAI/B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAN5B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoE;gBAGvE,UAAU,EAAE,IAAI,CAAC,cAAc,EAC/B,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,cAAc,YAAA,EACvB,SAAS,GAAE,MAAmC;IAG3D,UAAU,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAiBzD;AAED,qBAAa,sBAAsB;IAM/B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAR1B,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoE;IAC1F,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;gBAG9B,UAAU,EAAE,IAAI,CAAC,cAAc,EAC/B,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,EACnD,MAAM,CAAC,EAAE,cAAc,YAAA;IAG1C,KAAK,IAAI,IAAI;IAWb,IAAI,IAAI,IAAI;YAQE,aAAa;CAoB5B"}

package/esm/data-pump/nats-distribution.js

@@ -0,0 +1,126 @@
+import * as Nats from "nats";
+const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+export class NatsDistributionLeader {
+    constructor(connection, subject, logger, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
+        Object.defineProperty(this, "connection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: connection
+        });
+        Object.defineProperty(this, "subject", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: subject
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: logger
+        });
+        Object.defineProperty(this, "timeoutMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: timeoutMs
+        });
+        Object.defineProperty(this, "codec", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: Nats.JSONCodec()
+        });
+    }
+    async distribute(events) {
+        const deliveryId = crypto.randomUUID();
+        const request = { deliveryId, events };
+        this.logger?.debug("Distributing events via NATS", { deliveryId, count: events.length, subject: this.subject });
+        const response = await this.connection.request(this.subject, this.codec.encode(request), { timeout: this.timeoutMs });
+        const reply = this.codec.decode(response.data);
+        if (reply.status === "fail") {
+            throw new Error(`Worker failed to process events: ${reply.error ?? "unknown error"}`);
+        }
+    }
+}
+export class NatsDistributionWorker {
+    constructor(connection, subject, handler, logger) {
+        Object.defineProperty(this, "connection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: connection
+        });
+        Object.defineProperty(this, "subject", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: subject
+        });
+        Object.defineProperty(this, "handler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: handler
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: logger
+        });
+        Object.defineProperty(this, "subscription", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "codec", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: Nats.JSONCodec()
+        });
+        Object.defineProperty(this, "queueGroup", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "data-pump-workers"
+        });
+    }
+    start() {
+        this.logger?.info("Starting NATS distribution worker", { subject: this.subject, queue: this.queueGroup });
+        this.subscription = this.connection.subscribe(this.subject, {
+            queue: this.queueGroup,
+            callback: (_err, msg) => {
+                void this.handleMessage(msg);
+            },
+        });
+    }
+    stop() {
+        if (this.subscription) {
+            this.logger?.debug("Stopping NATS distribution worker");
+            this.subscription.unsubscribe();
+            this.subscription = undefined;
+        }
+    }
+    async handleMessage(msg) {
+        try {
+            const request = this.codec.decode(msg.data);
+            this.logger?.debug("Received events from leader via NATS", {
+                deliveryId: request.deliveryId,
+                count: request.events.length,
+            });
+            await this.handler(request.events);
+            const reply = { status: "ack" };
+            msg.respond(this.codec.encode(reply));
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : "unknown error";
+            this.logger?.error("Worker failed to process events via NATS", { error: errorMessage });
+            const reply = { status: "fail", error: errorMessage };
+            msg.respond(this.codec.encode(reply));
+        }
+    }
+}

package/esm/data-pump/notifier.d.ts

@@ -1,8 +1,10 @@
+import { NatsConnectionManager } from "./nats-connection.js";
 import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreLogger } from "./types.js";
 export interface FlowcoreNotifierOptions {
     dataSource: FlowcoreDataPumpDataSource;
     auth: FlowcoreDataPumpAuth;
     natsServers?: string[];
+    natsConnectionManager?: NatsConnectionManager;
     pollerIntervalMs?: number;
     timeoutMs?: number;
     logger?: FlowcoreLogger;
@@ -12,7 +14,7 @@ export interface FlowcoreNotifierOptions {
 export declare class FlowcoreNotifier {
     private readonly options;
     private dataSource;
-    private nats?;
+    private natsManager?;
     private subject?;
     private notificationClient?;
     private eventResolver?;

package/esm/data-pump/notifier.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"notifier.d.ts","sourceRoot":"","sources":["../../src/data-pump/notifier.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAKlG,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,0BAA0B,CAAA;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,aAAa,CAAC,EAAE,OAAO,CAAA;CACxB;AAED,qBAAa,gBAAgB;IASf,OAAO,CAAC,QAAQ,CAAC,OAAO;IARpC,OAAO,CAAC,UAAU,CAAoB;IACtC,OAAO,CAAC,IAAI,CAAC,CAAqB;IAClC,OAAO,CAAC,OAAO,CAAC,CAA4B;IAC5C,OAAO,CAAC,kBAAkB,CAAC,CAAoB;IAC/C,OAAO,CAAC,aAAa,CAAC,CAAY;IAElC,OAAO,CAAC,KAAK,CAAC,CAAK;gBAEU,OAAO,EAAE,uBAAuB;IAStD,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW;YASlB,UAAU;YAUV,QAAQ;IAuCtB,OAAO,CAAC,gBAAgB;YAMV,aAAa;IAmC3B,OAAO,CAAC,aAAa;CAgBtB"}
+{"version":3,"file":"notifier.d.ts","sourceRoot":"","sources":["../../src/data-pump/notifier.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAA;AAC5D,OAAO,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAKlG,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,0BAA0B,CAAA;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,qBAAqB,CAAC,EAAE,qBAAqB,CAAA;IAC7C,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,aAAa,CAAC,EAAE,OAAO,CAAA;CACxB;AAED,qBAAa,gBAAgB;IASf,OAAO,CAAC,QAAQ,CAAC,OAAO;IARpC,OAAO,CAAC,UAAU,CAAoB;IACtC,OAAO,CAAC,WAAW,CAAC,CAAuB;IAC3C,OAAO,CAAC,OAAO,CAAC,CAA4B;IAC5C,OAAO,CAAC,kBAAkB,CAAC,CAAoB;IAC/C,OAAO,CAAC,aAAa,CAAC,CAAY;IAElC,OAAO,CAAC,KAAK,CAAC,CAAK;gBAEU,OAAO,EAAE,uBAAuB;IAetD,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW;YASlB,UAAU;YAUV,QAAQ;IAoCtB,OAAO,CAAC,gBAAgB;YAMV,aAAa;IAmC3B,OAAO,CAAC,aAAa;CAgBtB"}

package/esm/data-pump/notifier.js

@@ -1,7 +1,7 @@
-import { NotificationClient } from "../deps/jsr.io/@flowcore/sdk/1.70.0/src/mod.js";
-import * as Nats from "nats";
+import { NotificationClient } from "../deps/jsr.io/@flowcore/sdk/1.78.0/src/mod.js";
 import { Subject } from "rxjs";
 import { FlowcoreDataSource } from "./data-source.js";
+import { NatsConnectionManager } from "./nats-connection.js";
 import { noOpLogger } from "./no-op-logger.js";
 const DEFAULT_TIMEOUT_MS = 20_000;
 export class FlowcoreNotifier {
@@ -18,7 +18,7 @@ export class FlowcoreNotifier {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "nats", {
+        Object.defineProperty(this, "natsManager", {
             enumerable: true,
             configurable: true,
             writable: true,
@@ -55,6 +55,12 @@ export class FlowcoreNotifier {
             directMode: this.options.directMode,
             noTranslation: this.options.noTranslation,
         });
+        if (this.options.natsConnectionManager) {
+            this.natsManager = this.options.natsConnectionManager;
+        }
+        else if (this.options.natsServers) {
+            this.natsManager = new NatsConnectionManager(this.options.natsServers, this.options.logger);
+        }
     }
     wait(signal) {
         if (this.options.natsServers) {
@@ -76,9 +82,7 @@ export class FlowcoreNotifier {
     }
     async waitNats(signal) {
         this.options.logger?.debug("Waiting for nats");
-        if (!this.nats) {
-            this.nats = await Nats.connect({ servers: this.options.natsServers });
-        }
+        const nats = await this.natsManager.connect();
         const dataCoreId = await this.dataSource.getDataCoreId();
         const topics = this.dataSource.eventTypes.map((eventType) => `stored.event.notify.0.${dataCoreId}.${this.dataSource.flowType}.${eventType}`);
         const promise = new Promise((resolve) => {
@@ -86,7 +90,7 @@ export class FlowcoreNotifier {
         });
         const subscriptions = [];
         for (const topic of topics) {
-            subscriptions.push(this.nats.subscribe(topic, {
+            subscriptions.push(nats.subscribe(topic, {
                 callback: () => {
                     this.options.logger?.debug(`Received event from nats: ${topic}`);
                     this.eventResolver?.();
@@ -101,8 +105,6 @@ export class FlowcoreNotifier {
         for (const subscription of subscriptions) {
             subscription.unsubscribe();
         }
-        this.nats?.close();
-        this.nats = undefined;
     }
     onWebSocketEvent(event) {
         if (this.dataSource.eventTypes.includes(event.data.eventType)) {

package/esm/data-pump/pulse.d.ts

@@ -0,0 +1,33 @@
+import type { FlowcoreDataPumpAuth, FlowcoreLogger } from "./types.js";
+export interface PulseEmitterOptions {
+    url: string;
+    intervalMs?: number;
+    auth: FlowcoreDataPumpAuth;
+    logger?: FlowcoreLogger;
+}
+export interface PulseSnapshot {
+    pathwayId: string;
+    flowType: string;
+    timeBucket: string;
+    eventId: string | undefined;
+    isLive: boolean;
+    bufferDepth: number;
+    bufferReserved: number;
+    bufferSizeBytes: number;
+    acknowledgedTotal: number;
+    failedTotal: number;
+    pulledTotal: number;
+    uptimeMs: number;
+}
+export declare class PulseEmitter {
+    private readonly options;
+    private readonly getSnapshot;
+    private interval;
+    private readonly intervalMs;
+    private readonly logger?;
+    constructor(options: PulseEmitterOptions, getSnapshot: () => PulseSnapshot | null);
+    start(): void;
+    stop(): void;
+    private emit;
+}
+//# sourceMappingURL=pulse.d.ts.map

package/esm/data-pump/pulse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pulse.d.ts","sourceRoot":"","sources":["../../src/data-pump/pulse.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAGtE,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,oBAAoB,CAAA;IAC1B,MAAM,CAAC,EAAE,cAAc,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAA;IAC3B,MAAM,EAAE,OAAO,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;IACzB,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,qBAAa,YAAY;IAMrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAN9B,OAAO,CAAC,QAAQ,CAA8C;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB;gBAGrB,OAAO,EAAE,mBAAmB,EAC5B,WAAW,EAAE,MAAM,aAAa,GAAG,IAAI;IAM1D,KAAK,IAAI,IAAI;IAUb,IAAI,IAAI,IAAI;YAOE,IAAI;CA2BnB"}