@flowcore/data-pump 0.22.1 → 0.23.0

package/dist/mod.d.ts

@@ -1,5 +1,5 @@
 import { FlowcoreEvent, FlowcoreClient, EventListOutput } from '@flowcore/sdk';
-import PromClient from 'prom-client';
+import { Registry, Gauge, Counter, Histogram } from 'prom-client';
 import * as Nats from 'nats';
 
 type FlowcoreDataPumpAuth = {
@@ -89,6 +89,8 @@ declare class FlowcoreDataSource {
     protected eventTypeIds?: string[];
     /** Cached time buckets */
     protected timeBuckets?: string[];
+    /** Index for each cached bucket. */
+    private timeBucketIndexes;
     /**
      * Creates a new FlowcoreDataSource instance
      * @param options - Configuration options for the data source
@@ -141,6 +143,8 @@ declare class FlowcoreDataSource {
      * @returns Promise that resolves to an array of time bucket strings
      */
     getTimeBuckets(force?: boolean): Promise<string[]>;
+    private lowerBoundTimeBucket;
+    private upperBoundTimeBucket;
     /**
      * Gets the next time bucket after the specified time bucket
      * @param timeBucket - The reference time bucket
@@ -186,6 +190,8 @@ interface PulseSnapshot {
     timeBucket: string;
     eventId: string | undefined;
     isLive: boolean;
+    /** Delivery to the processor is paused. The pump is still fetching and alive. */
+    paused: boolean;
     bufferDepth: number;
     bufferReserved: number;
     bufferSizeBytes: number;
@@ -237,6 +243,14 @@ interface FlowcoreDataPumpOptions {
     notifier?: FlowcoreDataPumpNotifierOptions;
     logger?: FlowcoreLogger;
     stopAt?: Date;
+    /**
+     * Start the pump with delivery already paused.
+     *
+     * Use this to restore a pause that is stored somewhere durable. Calling `pause()`
+     * after `start()` would let the pump deliver events in the gap between the two.
+     * Ignored when no `processor` is configured, for the same reason as {@link FlowcoreDataPump.pause}.
+     */
+    paused?: boolean;
     baseUrlOverride?: string;
     noTranslation?: boolean;
     directMode?: boolean;
@@ -259,7 +273,12 @@ declare class FlowcoreDataPump {
     private readonly logger?;
     private nextCursor?;
     private running;
+    private paused;
     private restartTo?;
+    private processLoopGeneration;
+    private activeProcessLoopGeneration?;
+    private processLoopBackoffGeneration?;
+    private processLoopRestartTimer?;
     private abortController?;
     private buffer;
     private bufferState;
@@ -273,30 +292,75 @@ declare class FlowcoreDataPump {
     private pulledCount;
     private processLoopRestartAttempts;
     private mainLoopRestartAttempts;
+    private readonly replayObserver;
+    private readonly bufferStats;
+    private bufferReservedCount;
+    private bufferSizeBytes;
+    private gaugePublicationScheduled;
     private constructor();
+    /**
+     * Whether delivery to the processor is currently paused.
+     * A paused pump is still running: it fetches, buffers and pulses.
+     */
+    get isPaused(): boolean;
     get isRunning(): boolean;
     getSnapshot(): PulseSnapshot | null;
     static create(options: FlowcoreDataPumpOptions, dataSourceOverride?: FlowcoreDataSource): FlowcoreDataPump;
     start(callback?: (error?: Error) => void): Promise<void>;
     private startMainLoop;
     restart(state: FlowcoreDataPumpState, stopAt?: Date | null): void;
+    /**
+     * Pause delivery to the processor.
+     *
+     * The fetch loop keeps running and tops the buffer up to `bufferSize`, then blocks on
+     * normal backpressure. The buffer, the cursor and the pulse emitter are untouched, so
+     * the control plane still sees a live pump. An in-flight batch finishes its handler
+     * and acknowledges, so the checkpoint stays accurate and nothing is redelivered
+     * needlessly.
+     *
+     * Idempotent. A paused pump holds up to `bufferSize` events in memory.
+     */
+    pause(): void;
+    /**
+     * Resume delivery to the processor from the exact position where {@link pause} stopped it.
+     * Idempotent.
+     */
+    resume(): void;
     stop(isRestart?: boolean): void;
     private updateState;
     private loop;
     reserve(amount: number): Promise<FlowcoreEvent[]>;
+    private reserveInternal;
     onFinalyFailed(handler: (events: FlowcoreEvent[]) => Promise<void> | void): void;
     acknowledge(eventIds: string[]): Promise<void>;
     fail(eventIds: string[]): Promise<void>;
     private reOpen;
+    /**
+     * Guarantee a live process loop whenever the pump is running with a
+     * processor. Called from the fetch loop, so a delivery loop that exited for
+     * any reason — most importantly a restart, which clears `running` while the
+     * loop is mid-batch — comes back within one fetch iteration instead of
+     * leaving a pump that pulls but never delivers.
+     */
+    private ensureProcessLoop;
     private startProcessLoop;
+    private isCurrentProcessLoop;
     private processLoop;
+    private addEventsToBuffer;
+    private updateReservedStats;
+    private removeFromBufferStats;
+    private resetBufferStats;
     private updateMetricsGauges;
+    private publishMetricsGauges;
     private incMetricsCounter;
-    private waiterEvents?;
+    private publicEventWaiter?;
+    private readonly processEventWaiters;
+    private notifyEventWaiters;
     private waitForEvents;
     private waiterBufferThreshold?;
     private waitForBufferThreshold;
     private waiterBufferEmpty?;
+    private notifyBufferEmpty;
     private waitForBufferEmpty;
 }
 
@@ -321,6 +385,7 @@ declare class FlowcoreDataPumpCluster {
     private readonly logger?;
     private running;
     private isLeader;
+    private paused;
     private pump?;
     private leaderConnection?;
     private workers;
@@ -372,6 +437,21 @@ declare class FlowcoreDataPumpCluster {
      */
     handleConnection(ws: WebSocket): void;
     start(): Promise<void>;
+    /**
+     * Whether delivery is paused across the cluster.
+     * The flag lives on the cluster, so it survives a leader change.
+     */
+    get isPaused(): boolean;
+    /**
+     * Pause delivery. A leader applies it to its pump immediately. A follower records it,
+     * so the pause is re-applied if this instance later becomes leader.
+     *
+     * The flag is in-memory. It does NOT survive a process restart or a full rolling
+     * deploy — persist it in the coordinator or the control plane if you need that.
+     */
+    pause(): void;
+    /** Resume delivery from the position where {@link pause} stopped it. */
+    resume(): void;
     stop(): Promise<void>;
     private startHeartbeat;
     private startElectionLoop;
@@ -392,22 +472,73 @@ declare class FlowcoreDataPumpCluster {
     private handleWorkerMessage;
 }
 
-declare const dataPumpPromRegistry: PromClient.Registry<"text/plain; version=0.0.4; charset=utf-8">;
+declare const dataPumpPromRegistry: Registry<"text/plain; version=0.0.4; charset=utf-8">;
+declare const REPLAY_DURATION_BUCKETS_SECONDS: readonly [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60, 120];
+declare const REPLAY_EVENT_COUNT_BUCKETS: readonly [1, 10, 50, 100, 250, 500, 1000, 2000, 5000, 10000];
+declare const REPLAY_BATCH_SIZE_BUCKETS: readonly ["empty", "1", "2-10", "11-100", "101-1000", "1001+"];
+declare const REPLAY_FETCH_RESULTS: readonly ["success", "no_events", "error"];
+declare const REPLAY_STAGE_RESULTS: readonly ["success", "error"];
+declare const REPLAY_IDLE_REASONS: readonly ["no_events", "buffer_full", "waiting_for_events"];
+type ReplayBatchSizeBucket = (typeof REPLAY_BATCH_SIZE_BUCKETS)[number];
+type ReplayIdleReason = (typeof REPLAY_IDLE_REASONS)[number];
+type ReplaySourceLabels = {
+    tenant: string;
+    data_core: string;
+    flow_type: string;
+};
+declare function replayBatchSizeBucket(size: number): ReplayBatchSizeBucket;
+declare function createDataPumpMetrics(registry: Registry, includeDefaultRegistry?: boolean): {
+    bufferEventCountGauge: Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    bufferReservedEventCountGauge: Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    bufferSizeBytesGauge: Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsAcknowledgedCounter: Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsFailedCounter: Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsPulledSizeBytesCounter: Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    sdkCommandsCounter: Counter<"command">;
+    replayFetchDuration: Histogram<"tenant" | "data_core" | "flow_type" | "result">;
+    replayFetchEvents: Histogram<"tenant" | "data_core" | "flow_type">;
+    replayHandlerDuration: Histogram<"tenant" | "data_core" | "flow_type" | "result" | "batch_size_bucket">;
+    replayAcknowledgementDuration: Histogram<"tenant" | "data_core" | "flow_type" | "result">;
+    replayCheckpointDuration: Histogram<"tenant" | "data_core" | "flow_type" | "result">;
+    replayIdleDuration: Histogram<"tenant" | "data_core" | "flow_type" | "reason">;
+};
+type DataPumpMetrics = ReturnType<typeof createDataPumpMetrics>;
 declare const metrics: {
-    bufferEventCountGauge: PromClient.Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
-    bufferReservedEventCountGauge: PromClient.Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
-    bufferSizeBytesGauge: PromClient.Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
-    eventsAcknowledgedCounter: PromClient.Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
-    eventsFailedCounter: PromClient.Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
-    eventsPulledSizeBytesCounter: PromClient.Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
-    sdkCommandsCounter: PromClient.Counter<"command">;
+    bufferEventCountGauge: Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    bufferReservedEventCountGauge: Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    bufferSizeBytesGauge: Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsAcknowledgedCounter: Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsFailedCounter: Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsPulledSizeBytesCounter: Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    sdkCommandsCounter: Counter<"command">;
+    replayFetchDuration: Histogram<"tenant" | "data_core" | "flow_type" | "result">;
+    replayFetchEvents: Histogram<"tenant" | "data_core" | "flow_type">;
+    replayHandlerDuration: Histogram<"tenant" | "data_core" | "flow_type" | "result" | "batch_size_bucket">;
+    replayAcknowledgementDuration: Histogram<"tenant" | "data_core" | "flow_type" | "result">;
+    replayCheckpointDuration: Histogram<"tenant" | "data_core" | "flow_type" | "result">;
+    replayIdleDuration: Histogram<"tenant" | "data_core" | "flow_type" | "reason">;
 };
+declare class ReplayStageObserver {
+    private readonly stageMetrics;
+    private readonly source;
+    private readonly now;
+    constructor(stageMetrics: DataPumpMetrics, source: ReplaySourceLabels, now?: () => number);
+    observeFetch<T extends {
+        events: unknown[];
+    }>(operation: () => Promise<T>): Promise<T>;
+    observeHandler<T>(events: unknown[], operation: () => Promise<T>): Promise<T>;
+    observeAcknowledgement<T>(operation: () => T): T;
+    observeCheckpoint<T>(operation: () => Promise<T> | T): Promise<T>;
+    observeIdle<T>(reason: ReplayIdleReason, operation: () => Promise<T>): Promise<T>;
+    private observeResult;
+    private elapsedSeconds;
+}
 declare const clusterMetrics: {
-    activeWorkersGauge: PromClient.Gauge<string>;
-    leaderStatusGauge: PromClient.Gauge<string>;
-    eventsDistributedCounter: PromClient.Counter<string>;
-    workerAcksCounter: PromClient.Counter<string>;
-    workerFailsCounter: PromClient.Counter<string>;
+    activeWorkersGauge: Gauge<string>;
+    leaderStatusGauge: Gauge<string>;
+    eventsDistributedCounter: Counter<string>;
+    workerAcksCounter: Counter<string>;
+    workerFailsCounter: Counter<string>;
 };
 
 declare class NatsConnectionManager {
@@ -514,4 +645,4 @@ declare class DeliveryTracker {
 
 declare const noOpLogger: FlowcoreLogger;
 
-export { DeliveryTracker, FlowcoreDataPump, type FlowcoreDataPumpAuth, FlowcoreDataPumpCluster, type FlowcoreDataPumpClusterOptions, type FlowcoreDataPumpCoordinator, type FlowcoreDataPumpDataSource, type FlowcoreDataPumpOptions, type FlowcoreDataPumpProcessor, type FlowcoreDataPumpState, type FlowcoreDataPumpStateManager, FlowcoreDataSource, type FlowcoreDataSourceOptions, type FlowcoreLogger, NatsConnectionManager, NatsDistributionLeader, type NatsDistributionReply, type NatsDistributionRequest, NatsDistributionWorker, type PendingDelivery, PulseEmitter, type PulseEmitterOptions, type PulseLogLevel, type PulseSnapshot, type WsAckMessage, WsConnection, type WsConnectionOptions, type WsEventsMessage, type WsFailMessage, type WsMessage, type WsPingMessage, type WsPongMessage, clusterMetrics, dataPumpPromRegistry, deserializeMessage, metrics, noOpLogger, serializeMessage };
+export { type DataPumpMetrics, DeliveryTracker, FlowcoreDataPump, type FlowcoreDataPumpAuth, FlowcoreDataPumpCluster, type FlowcoreDataPumpClusterOptions, type FlowcoreDataPumpCoordinator, type FlowcoreDataPumpDataSource, type FlowcoreDataPumpOptions, type FlowcoreDataPumpProcessor, type FlowcoreDataPumpState, type FlowcoreDataPumpStateManager, FlowcoreDataSource, type FlowcoreDataSourceOptions, type FlowcoreLogger, NatsConnectionManager, NatsDistributionLeader, type NatsDistributionReply, type NatsDistributionRequest, NatsDistributionWorker, type PendingDelivery, PulseEmitter, type PulseEmitterOptions, type PulseLogLevel, type PulseSnapshot, REPLAY_BATCH_SIZE_BUCKETS, REPLAY_DURATION_BUCKETS_SECONDS, REPLAY_EVENT_COUNT_BUCKETS, REPLAY_FETCH_RESULTS, REPLAY_IDLE_REASONS, REPLAY_STAGE_RESULTS, type ReplayBatchSizeBucket, type ReplayIdleReason, type ReplaySourceLabels, ReplayStageObserver, type WsAckMessage, WsConnection, type WsConnectionOptions, type WsEventsMessage, type WsFailMessage, type WsMessage, type WsPingMessage, type WsPongMessage, clusterMetrics, createDataPumpMetrics, dataPumpPromRegistry, deserializeMessage, metrics, noOpLogger, replayBatchSizeBucket, serializeMessage };