@flowcore/data-pump 0.22.1 → 0.23.0

package/dist/mod.js

@@ -17,58 +17,227 @@ import {
 import { FlowcoreClient } from "@flowcore/sdk";
 
 // src/data-pump/metrics.ts
-import PromClient, { Counter, Gauge } from "prom-client";
-var dataPumpPromRegistry = new PromClient.Registry();
-var bufferEventCountGauge = new Gauge({
-  name: "flowcore_data_pump_buffer_events_gauge",
-  help: "The number of events in the buffer",
-  labelNames: ["tenant", "data_core", "flow_type", "event_type"]
-});
-var bufferReservedEventCountGauge = new Gauge({
-  name: "flowcore_data_pump_buffer_reserved_events_gauge",
-  help: "The number of reserved events in the buffer",
-  labelNames: ["tenant", "data_core", "flow_type", "event_type"]
-});
-var bufferSizeBytesGauge = new Gauge({
-  name: "flowcore_data_pump_buffer_size_bytes_gauge",
-  help: "The size of the buffer in bytes",
-  labelNames: ["tenant", "data_core", "flow_type", "event_type"]
-});
-var eventsAcknowledgedCounter = new Counter({
-  name: "flowcore_data_pump_events_acknowledged_counter",
-  help: "The number of events acknowledged",
-  labelNames: ["tenant", "data_core", "flow_type", "event_type"]
-});
-var eventsFailedCounter = new Counter({
-  name: "flowcore_data_pump_events_failed_counter",
-  help: "The number of events failed",
-  labelNames: ["tenant", "data_core", "flow_type", "event_type"]
-});
-var eventsPulledSizeBytesCounter = new Counter({
-  name: "flowcore_data_pump_events_pulled_size_bytes_counter",
-  help: "The size of the events pulled in bytes",
-  labelNames: ["tenant", "data_core", "flow_type", "event_type"]
-});
-var sdkCommandsCounter = new Counter({
-  name: "flowcore_data_pump_sdk_commands_counter",
-  help: "The number of SDK commands",
-  labelNames: ["command"]
-});
-dataPumpPromRegistry.registerMetric(bufferEventCountGauge);
-dataPumpPromRegistry.registerMetric(bufferReservedEventCountGauge);
-dataPumpPromRegistry.registerMetric(bufferSizeBytesGauge);
-dataPumpPromRegistry.registerMetric(eventsAcknowledgedCounter);
-dataPumpPromRegistry.registerMetric(eventsFailedCounter);
-dataPumpPromRegistry.registerMetric(eventsPulledSizeBytesCounter);
-dataPumpPromRegistry.registerMetric(sdkCommandsCounter);
-var metrics = {
-  bufferEventCountGauge,
-  bufferReservedEventCountGauge,
-  bufferSizeBytesGauge,
-  eventsAcknowledgedCounter,
-  eventsFailedCounter,
-  eventsPulledSizeBytesCounter,
-  sdkCommandsCounter
+import { Counter, Gauge, Histogram, register as defaultPromRegistry, Registry } from "prom-client";
+var dataPumpPromRegistry = new Registry();
+var SOURCE_LABELS = ["tenant", "data_core", "flow_type"];
+var EVENT_SOURCE_LABELS = [...SOURCE_LABELS, "event_type"];
+var REPLAY_DURATION_BUCKETS_SECONDS = [
+  5e-3,
+  0.01,
+  0.025,
+  0.05,
+  0.1,
+  0.25,
+  0.5,
+  1,
+  2,
+  5,
+  10,
+  20,
+  30,
+  60,
+  120
+];
+var REPLAY_EVENT_COUNT_BUCKETS = [1, 10, 50, 100, 250, 500, 1e3, 2e3, 5e3, 1e4];
+var REPLAY_BATCH_SIZE_BUCKETS = ["empty", "1", "2-10", "11-100", "101-1000", "1001+"];
+var REPLAY_FETCH_RESULTS = ["success", "no_events", "error"];
+var REPLAY_STAGE_RESULTS = ["success", "error"];
+var REPLAY_IDLE_REASONS = ["no_events", "buffer_full", "waiting_for_events"];
+function replayBatchSizeBucket(size) {
+  if (size <= 0) return "empty";
+  if (size === 1) return "1";
+  if (size <= 10) return "2-10";
+  if (size <= 100) return "11-100";
+  if (size <= 1e3) return "101-1000";
+  return "1001+";
+}
+function createDataPumpMetrics(registry, includeDefaultRegistry = false) {
+  const registers = includeDefaultRegistry ? [registry, defaultPromRegistry] : [registry];
+  const bufferEventCountGauge = new Gauge({
+    name: "flowcore_data_pump_buffer_events_gauge",
+    help: "The number of events in the buffer",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const bufferReservedEventCountGauge = new Gauge({
+    name: "flowcore_data_pump_buffer_reserved_events_gauge",
+    help: "The number of reserved events in the buffer",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const bufferSizeBytesGauge = new Gauge({
+    name: "flowcore_data_pump_buffer_size_bytes_gauge",
+    help: "The size of the buffer in bytes",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const eventsAcknowledgedCounter = new Counter({
+    name: "flowcore_data_pump_events_acknowledged_counter",
+    help: "The number of events acknowledged",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const eventsFailedCounter = new Counter({
+    name: "flowcore_data_pump_events_failed_counter",
+    help: "The number of events failed",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const eventsPulledSizeBytesCounter = new Counter({
+    name: "flowcore_data_pump_events_pulled_size_bytes_counter",
+    help: "The size of the events pulled in bytes",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const sdkCommandsCounter = new Counter({
+    name: "flowcore_data_pump_sdk_commands_counter",
+    help: "The number of SDK commands",
+    labelNames: ["command"],
+    registers
+  });
+  const replayFetchDuration = new Histogram({
+    name: "flowcore_data_pump_replay_fetch_duration_seconds",
+    help: "Replay event fetch duration in seconds",
+    labelNames: [...SOURCE_LABELS, "result"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS],
+    registers
+  });
+  const replayFetchEvents = new Histogram({
+    name: "flowcore_data_pump_replay_fetch_events",
+    help: "Number of events returned by a replay fetch",
+    labelNames: SOURCE_LABELS,
+    buckets: [...REPLAY_EVENT_COUNT_BUCKETS],
+    registers
+  });
+  const replayHandlerDuration = new Histogram({
+    name: "flowcore_data_pump_replay_handler_duration_seconds",
+    help: "Replay handler duration in seconds",
+    labelNames: [...SOURCE_LABELS, "result", "batch_size_bucket"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS],
+    registers
+  });
+  const replayAcknowledgementDuration = new Histogram({
+    name: "flowcore_data_pump_replay_acknowledgement_duration_seconds",
+    help: "Replay acknowledgement duration in seconds, excluding checkpointing",
+    labelNames: [...SOURCE_LABELS, "result"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS],
+    registers
+  });
+  const replayCheckpointDuration = new Histogram({
+    name: "flowcore_data_pump_replay_checkpoint_duration_seconds",
+    help: "Replay state checkpoint duration in seconds",
+    labelNames: [...SOURCE_LABELS, "result"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS],
+    registers
+  });
+  const replayIdleDuration = new Histogram({
+    name: "flowcore_data_pump_replay_idle_duration_seconds",
+    help: "Time the replay loop spends idle",
+    labelNames: [...SOURCE_LABELS, "reason"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS, 300],
+    registers
+  });
+  return {
+    bufferEventCountGauge,
+    bufferReservedEventCountGauge,
+    bufferSizeBytesGauge,
+    eventsAcknowledgedCounter,
+    eventsFailedCounter,
+    eventsPulledSizeBytesCounter,
+    sdkCommandsCounter,
+    replayFetchDuration,
+    replayFetchEvents,
+    replayHandlerDuration,
+    replayAcknowledgementDuration,
+    replayCheckpointDuration,
+    replayIdleDuration
+  };
+}
+var metrics = createDataPumpMetrics(dataPumpPromRegistry, true);
+var ReplayStageObserver = class {
+  constructor(stageMetrics, source, now = () => performance.now()) {
+    this.stageMetrics = stageMetrics;
+    this.source = source;
+    this.now = now;
+  }
+  stageMetrics;
+  source;
+  now;
+  async observeFetch(operation) {
+    const startedAt = this.now();
+    try {
+      const result = await operation();
+      this.stageMetrics.replayFetchDuration.observe(
+        { ...this.source, result: result.events.length ? "success" : "no_events" },
+        this.elapsedSeconds(startedAt)
+      );
+      this.stageMetrics.replayFetchEvents.observe(this.source, result.events.length);
+      return result;
+    } catch (error) {
+      this.stageMetrics.replayFetchDuration.observe({ ...this.source, result: "error" }, this.elapsedSeconds(startedAt));
+      throw error;
+    }
+  }
+  async observeHandler(events, operation) {
+    const startedAt = this.now();
+    const batchSize = replayBatchSizeBucket(events.length);
+    try {
+      const result = await operation();
+      this.stageMetrics.replayHandlerDuration.observe(
+        { ...this.source, result: "success", batch_size_bucket: batchSize },
+        this.elapsedSeconds(startedAt)
+      );
+      return result;
+    } catch (error) {
+      this.stageMetrics.replayHandlerDuration.observe(
+        { ...this.source, result: "error", batch_size_bucket: batchSize },
+        this.elapsedSeconds(startedAt)
+      );
+      throw error;
+    }
+  }
+  observeAcknowledgement(operation) {
+    const startedAt = this.now();
+    try {
+      const result = operation();
+      this.stageMetrics.replayAcknowledgementDuration.observe(
+        { ...this.source, result: "success" },
+        this.elapsedSeconds(startedAt)
+      );
+      return result;
+    } catch (error) {
+      this.stageMetrics.replayAcknowledgementDuration.observe(
+        { ...this.source, result: "error" },
+        this.elapsedSeconds(startedAt)
+      );
+      throw error;
+    }
+  }
+  observeCheckpoint(operation) {
+    return this.observeResult(this.stageMetrics.replayCheckpointDuration, operation);
+  }
+  async observeIdle(reason, operation) {
+    const startedAt = this.now();
+    try {
+      return await operation();
+    } finally {
+      this.stageMetrics.replayIdleDuration.observe({ ...this.source, reason }, this.elapsedSeconds(startedAt));
+    }
+  }
+  async observeResult(histogram, operation) {
+    const startedAt = this.now();
+    try {
+      const result = await operation();
+      histogram.observe({ ...this.source, result: "success" }, this.elapsedSeconds(startedAt));
+      return result;
+    } catch (error) {
+      histogram.observe({ ...this.source, result: "error" }, this.elapsedSeconds(startedAt));
+      throw error;
+    }
+  }
+  elapsedSeconds(startedAt) {
+    return Math.max(0, this.now() - startedAt) / 1e3;
+  }
 };
 var activeWorkersGauge = new Gauge({
   name: "flowcore_data_pump_cluster_active_workers_gauge",
@@ -145,6 +314,8 @@ var FlowcoreDataSource = class {
   eventTypeIds;
   /** Cached time buckets */
   timeBuckets;
+  /** Index for each cached bucket. */
+  timeBucketIndexes = /* @__PURE__ */ new Map();
   /**
    * Gets the tenant name from the data source configuration
    * @returns Tenant name
@@ -277,16 +448,46 @@ var FlowcoreDataSource = class {
           tenant: this.options.dataSource.tenant,
           eventTypeId: await this.getEventTypeIds(),
           cursor: cursor || void 0,
-          pageSize: 1e4
+          pageSize: 1e4,
+          order: "asc"
         }),
         this.options.directMode
       );
       timeBuckets.push(...result.timeBuckets);
       cursor = result.nextCursor;
     } while (cursor !== void 0);
-    this.timeBuckets = timeBuckets;
+    const normalizedTimeBuckets = [...new Set(timeBuckets.sort())];
+    this.timeBuckets = normalizedTimeBuckets;
+    this.timeBucketIndexes = /* @__PURE__ */ new Map();
+    for (let index = 0; index < normalizedTimeBuckets.length; index++) {
+      this.timeBucketIndexes.set(normalizedTimeBuckets[index], index);
+    }
     return this.timeBuckets;
   }
+  lowerBoundTimeBucket(timeBucket) {
+    const timeBuckets = this.timeBuckets ?? [];
+    const target = Number.parseFloat(timeBucket);
+    let low = 0;
+    let high = timeBuckets.length;
+    while (low < high) {
+      const middle = low + Math.floor((high - low) / 2);
+      if (Number.parseFloat(timeBuckets[middle]) < target) low = middle + 1;
+      else high = middle;
+    }
+    return low;
+  }
+  upperBoundTimeBucket(timeBucket) {
+    const timeBuckets = this.timeBuckets ?? [];
+    const target = Number.parseFloat(timeBucket);
+    let low = 0;
+    let high = timeBuckets.length;
+    while (low < high) {
+      const middle = low + Math.floor((high - low) / 2);
+      if (Number.parseFloat(timeBuckets[middle]) <= target) low = middle + 1;
+      else high = middle;
+    }
+    return low;
+  }
   /**
    * Gets the next time bucket after the specified time bucket
    * @param timeBucket - The reference time bucket
@@ -298,8 +499,8 @@ var FlowcoreDataSource = class {
       return null;
     }
     const timeBuckets = await this.getTimeBuckets();
-    const index = timeBuckets.indexOf(closestTimeBucket);
-    if (index === -1) {
+    const index = this.timeBucketIndexes.get(closestTimeBucket);
+    if (index === void 0) {
       throw new Error(`Could not get next timeBucket, timeBucket ${timeBucket} not found`);
     }
     return timeBuckets[index + 1] ?? null;
@@ -316,10 +517,14 @@ var FlowcoreDataSource = class {
     if (!timeBucket.match(/^\d{14}$/)) {
       throw new Error(`Invalid timebucket: ${timeBucket}`);
     }
+    if (!timeBuckets.length) {
+      return null;
+    }
     if (getBefore) {
-      return timeBuckets.findLast((t) => Number.parseFloat(t) <= Number.parseFloat(timeBucket)) ?? timeBuckets[timeBuckets.length - 1];
+      const index = this.upperBoundTimeBucket(timeBucket) - 1;
+      return timeBuckets[index] ?? timeBuckets[timeBuckets.length - 1];
     }
-    return timeBuckets.find((t) => Number.parseFloat(t) >= Number.parseFloat(timeBucket)) ?? timeBuckets[timeBuckets.length - 1];
+    return timeBuckets[this.lowerBoundTimeBucket(timeBucket)] ?? timeBuckets[timeBuckets.length - 1];
   }
   /**
    * Gets events from the data source starting from a specific state
@@ -620,6 +825,9 @@ var PulseEmitter = class {
       timeBucket: snapshot.timeBucket,
       eventId: snapshot.eventId ?? null,
       isLive: snapshot.isLive,
+      // Same widening as `sourceId` above: the CP accepts `paused`, the pinned sdk
+      // type does not declare it yet. The cast below absorbs it.
+      paused: snapshot.paused,
       buffer: {
         depth: snapshot.bufferDepth,
         reserved: snapshot.bufferReserved,
@@ -644,6 +852,7 @@ var PulseEmitter = class {
 };
 
 // src/data-pump/data-pump.ts
+var textEncoder = new TextEncoder();
 var FlowcoreDataPump = class _FlowcoreDataPump {
   constructor(dataSource, notifier, stateManager, options, logger) {
     this.dataSource = dataSource;
@@ -651,6 +860,14 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
     this.stateManager = stateManager;
     this.options = options;
     this.logger = logger;
+    this.replayObserver = new ReplayStageObserver(metrics, {
+      tenant: this.dataSource.tenant,
+      data_core: this.dataSource.dataCore,
+      flow_type: this.dataSource.flowType
+    });
+    for (const eventType of this.dataSource.eventTypes) {
+      this.bufferStats.set(eventType, { eventCount: 0, eventReservedCount: 0, eventSizeBytes: 0 });
+    }
     this.bufferState = {
       timeBucket: format(startOfHour(utc(/* @__PURE__ */ new Date())), "yyyyMMddHH0000"),
       eventId: TimeUuid.now().toString()
@@ -663,7 +880,18 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
   logger;
   nextCursor;
   running = false;
+  // Delivery pause. Independent of `running`: a paused pump keeps fetching, keeps its
+  // buffer, keeps its cursor and keeps emitting pulses — it only stops handing events
+  // to the processor. The flag is sticky across `restart()` and `stop()`/`start()`, so
+  // a repositioned or bounced pump stays paused until `resume()` is called.
+  paused = false;
   restartTo;
+  // Invalidates delivery work that crossed a stop/restart boundary. Event IDs
+  // can reappear during replay, so `running` alone cannot identify the owner.
+  processLoopGeneration = 0;
+  activeProcessLoopGeneration;
+  processLoopBackoffGeneration;
+  processLoopRestartTimer;
   abortController;
   buffer = [];
   bufferState;
@@ -677,13 +905,23 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
   pulledCount = 0;
   processLoopRestartAttempts = 0;
   mainLoopRestartAttempts = 0;
+  replayObserver;
+  bufferStats = /* @__PURE__ */ new Map();
+  bufferReservedCount = 0;
+  bufferSizeBytes = 0;
+  gaugePublicationScheduled = false;
+  /**
+   * Whether delivery to the processor is currently paused.
+   * A paused pump is still running: it fetches, buffers and pulses.
+   */
+  get isPaused() {
+    return this.paused;
+  }
   get isRunning() {
     return this.running;
   }
   getSnapshot() {
     if (!this.running) return null;
-    const reserved = this.buffer.filter((b) => b.status === "reserved").length;
-    const sizeBytes = this.buffer.reduce((sum, b) => sum + JSON.stringify(b.event.payload).length, 0);
     return {
       pathwayId: this.pulseEmitter ? "" : "",
       // set by caller
@@ -691,9 +929,10 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
       timeBucket: this.bufferState.timeBucket,
       eventId: this.bufferState.eventId,
       isLive: this.isLive,
+      paused: this.paused,
       bufferDepth: this.buffer.length,
-      bufferReserved: reserved,
-      bufferSizeBytes: sizeBytes,
+      bufferReserved: this.bufferReservedCount,
+      bufferSizeBytes: this.bufferSizeBytes,
       acknowledgedTotal: this.acknowledgedCount,
       failedTotal: this.failedCount,
       pulledTotal: this.pulledCount,
@@ -739,6 +978,9 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
       },
       options.logger
     );
+    if (options.paused && options.processor) {
+      pump.paused = true;
+    }
     if (options.pulse) {
       const pathwayId = options.pulse.pathwayId;
       const sourceId = options.pulse.sourceId;
@@ -771,7 +1013,7 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
     this.running = true;
     this.startedAt = Date.now();
     this.nextCursor = void 0;
-    this.updateMetricsGauges();
+    this.updateMetricsGauges(true);
     this.pulseEmitter?.start();
     const currentState = await this.stateManager.getState();
     const timeBucket = currentState ? await this.dataSource.getClosestTimeBucket(currentState.timeBucket) : format(startOfHour(utc(/* @__PURE__ */ new Date())), "yyyyMMddHH0000");
@@ -826,23 +1068,68 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
     if (stopAt !== void 0) {
       this.options.stopAt = stopAt ?? void 0;
     }
+    this.isLive = false;
     this.stop(true);
   }
+  /**
+   * Pause delivery to the processor.
+   *
+   * The fetch loop keeps running and tops the buffer up to `bufferSize`, then blocks on
+   * normal backpressure. The buffer, the cursor and the pulse emitter are untouched, so
+   * the control plane still sees a live pump. An in-flight batch finishes its handler
+   * and acknowledges, so the checkpoint stays accurate and nothing is redelivered
+   * needlessly.
+   *
+   * Idempotent. A paused pump holds up to `bufferSize` events in memory.
+   */
+  pause() {
+    if (!this.options.processor) {
+      this.logger?.warn("pause() ignored: this pump has no processor. Stop calling reserve() instead.");
+      return;
+    }
+    if (this.paused) return;
+    this.paused = true;
+    this.logger?.info("Data pump paused");
+    this.notifyEventWaiters(false);
+  }
+  /**
+   * Resume delivery to the processor from the exact position where {@link pause} stopped it.
+   * Idempotent.
+   */
+  resume() {
+    if (!this.paused) return;
+    this.paused = false;
+    if (this.processLoopRestartTimer) {
+      clearTimeout(this.processLoopRestartTimer);
+      this.processLoopRestartTimer = void 0;
+    }
+    this.processLoopBackoffGeneration = void 0;
+    this.processLoopRestartAttempts = 0;
+    this.logger?.info("Data pump resumed");
+    this.ensureProcessLoop();
+  }
   stop(isRestart = false) {
     this.running = false;
+    this.processLoopGeneration++;
+    this.activeProcessLoopGeneration = void 0;
+    this.processLoopBackoffGeneration = void 0;
+    if (this.processLoopRestartTimer) {
+      clearTimeout(this.processLoopRestartTimer);
+      this.processLoopRestartTimer = void 0;
+    }
     this.processLoopRestartAttempts = 0;
     this.mainLoopRestartAttempts = 0;
     this.buffer = [];
-    this.updateMetricsGauges();
+    this.resetBufferStats();
+    this.updateMetricsGauges(true);
     this.pulseEmitter?.stop();
     this.abortController?.abort();
     this.waiterBufferThreshold?.();
-    if (!isRestart) {
-      this.waiterEvents?.();
-    }
+    this.notifyEventWaiters(!isRestart);
   }
   updateState(eventId) {
-    if (!this.stateManager.setState) {
+    const stateManager = this.stateManager;
+    if (!stateManager.setState) {
       return;
     }
     const stateEventId = eventId ?? this.buffer[0]?.event.eventId;
@@ -851,23 +1138,26 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
     }
     const date = TimeUuid.fromString(stateEventId).getDate();
     const timeBucket = format(startOfHour(utc(date)), "yyyyMMddHH0000");
-    return this.stateManager.setState?.({ timeBucket, eventId: stateEventId });
+    return this.replayObserver.observeCheckpoint(() => stateManager.setState({ timeBucket, eventId: stateEventId }));
   }
   async loop() {
     do {
+      this.ensureProcessLoop();
       const amountToFetch = this.options.bufferSize - this.buffer.length;
       if (amountToFetch <= 0) {
         this.logger?.info("Buffer is full, waiting for space");
-        await this.waitForBufferThreshold();
+        await this.replayObserver.observeIdle("buffer_full", () => this.waitForBufferThreshold());
         continue;
       }
       this.logger?.debug(`fetching ${amountToFetch} events from ${this.bufferState.timeBucket}(${this.nextCursor})`);
-      const { events, nextCursor } = await this.dataSource.getEvents(
-        this.bufferState,
-        amountToFetch,
-        this.stopAtState?.eventId,
-        this.nextCursor,
-        this.options.includeSensitiveData
+      const { events, nextCursor } = await this.replayObserver.observeFetch(
+        () => this.dataSource.getEvents(
+          this.bufferState,
+          amountToFetch,
+          this.stopAtState?.eventId,
+          this.nextCursor,
+          this.options.includeSensitiveData
+        )
       );
       if (!this.running) {
         break;
@@ -875,10 +1165,10 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
       this.logger?.debug(`fetched ${events.length} events`);
       this.pulledCount += events.length;
       this.mainLoopRestartAttempts = 0;
-      this.buffer.push(...events.map((event) => ({ event, status: "open", deliveryCount: 0 })));
+      this.addEventsToBuffer(events);
       this.nextCursor = nextCursor;
       this.updateMetricsGauges();
-      events.length && this.waiterEvents?.();
+      events.length && this.notifyEventWaiters();
       this.bufferState.eventId = events[events.length - 1]?.eventId ?? this.bufferState.eventId;
       if (this.stopAtState?.timeBucket && this.bufferState.timeBucket >= this.stopAtState.timeBucket && !events.length) {
         this.logger?.info("Stopping at stopAt state");
@@ -900,9 +1190,12 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
             this.isLive = true;
             this.logger?.debug("Going live...");
             this.abortController = new AbortController();
-            await this.notifier.wait(this.abortController.signal);
+            await this.replayObserver.observeIdle("no_events", () => this.notifier.wait(this.abortController.signal));
           } else if (this.isLive) {
-            await new Promise((resolve) => setTimeout(resolve, 1e3));
+            await this.replayObserver.observeIdle(
+              "no_events",
+              () => new Promise((resolve) => setTimeout(resolve, 1e3))
+            );
           }
         }
       }
@@ -915,18 +1208,27 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
         this.bufferState = this.restartTo;
         this.restartTo = void 0;
         this.running = true;
+        this.ensureProcessLoop();
+        this.pulseEmitter?.start();
         return this.loop();
       } catch (error) {
         this.logger?.error("Failed to consume restartTo, dropping it", { error });
         this.restartTo = void 0;
+        this.notifyEventWaiters();
         return;
       }
     }
     this.logger?.info("Data pump stopped");
   }
   // #region Puller
-  async reserve(amount) {
-    if (!this.running) {
+  reserve(amount) {
+    return this.reserveInternal(amount);
+  }
+  async reserveInternal(amount, generation) {
+    if (!this.running || generation !== void 0 && generation !== this.processLoopGeneration) {
+      return [];
+    }
+    if (generation !== void 0 && this.paused) {
       return [];
     }
     const events = [];
@@ -936,23 +1238,27 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
         event.status = "reserved";
         event.deliveryId = deliveryId;
         event.deliveryCount++;
+        this.updateReservedStats(event, 1);
         events.push(event.event);
-        this.incMetricsCounter("pulled", event.event.eventType, JSON.stringify(event.event).length);
+        event.eventSizeBytes ??= textEncoder.encode(JSON.stringify(event.event)).byteLength;
+        this.incMetricsCounter("pulled", event.event.eventType, event.eventSizeBytes);
         if (events.length === amount) {
           break;
         }
       }
     }
     if (!events.length) {
-      await this.waitForEvents();
-      return this.reserve(amount);
+      await this.waitForEvents(generation);
+      return this.reserveInternal(amount, generation);
     }
     this.updateMetricsGauges();
     setTimeout(() => {
-      this.reOpen(
+      void this.reOpen(
         events.map((event) => event.eventId),
         deliveryId
-      );
+      ).catch((error) => {
+        this.logger?.error("Failed to reopen events after acknowledgement timeout", { error });
+      });
     }, this.options.achknowledgeTimeoutMs);
     return events;
   }
@@ -963,22 +1269,28 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
     if (!this.running) {
       return;
     }
-    const lastEventInBuffer = this.buffer[this.buffer.length - 1];
-    this.buffer = this.buffer.filter((event) => {
-      if (eventIds.includes(event.event.eventId)) {
-        this.incMetricsCounter("acknowledged", event.event.eventType, 1);
-        this.acknowledgedCount++;
-        return false;
+    const eventIdSet = new Set(eventIds);
+    const checkpointEventId = this.replayObserver.observeAcknowledgement(() => {
+      const lastEventInBuffer = this.buffer[this.buffer.length - 1];
+      this.buffer = this.buffer.filter((event) => {
+        if (eventIdSet.has(event.event.eventId)) {
+          this.incMetricsCounter("acknowledged", event.event.eventType, 1);
+          this.acknowledgedCount++;
+          this.removeFromBufferStats(event);
+          return false;
+        }
+        return true;
+      });
+      if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+        this.waiterBufferThreshold?.();
       }
-      return true;
+      return this.buffer.length ? void 0 : lastEventInBuffer?.event.eventId;
     });
-    if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
-      this.waiterBufferThreshold?.();
-    }
-    await this.updateState(this.buffer.length ? void 0 : lastEventInBuffer?.event.eventId);
     this.updateMetricsGauges();
-    if (!this.buffer.length) {
-      this.waiterBufferEmpty?.();
+    try {
+      await this.updateState(checkpointEventId);
+    } finally {
+      this.notifyBufferEmpty();
     }
   }
   async fail(eventIds) {
@@ -986,84 +1298,140 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
       return;
     }
     const lastEventInBuffer = this.buffer[this.buffer.length - 1];
+    const eventIdSet = new Set(eventIds);
     const failedEvents = [];
     this.buffer = this.buffer.filter((event) => {
-      if (eventIds.includes(event.event.eventId)) {
+      if (eventIdSet.has(event.event.eventId)) {
         this.incMetricsCounter("failed", event.event.eventType, 1);
         this.failedCount++;
         failedEvents.push(event.event);
+        this.removeFromBufferStats(event);
         return false;
       }
       return true;
     });
     this.logger?.info(`Failed ${failedEvents.length} events`);
-    void this.options.processor?.failedHandler?.(failedEvents);
     if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
       this.waiterBufferThreshold?.();
     }
-    await this.updateState(this.buffer.length ? void 0 : lastEventInBuffer?.event.eventId);
-    if (!this.buffer.length) {
-      this.waiterBufferEmpty?.();
+    this.updateMetricsGauges();
+    try {
+      await this.options.processor?.failedHandler?.(failedEvents);
+      await this.updateState(this.buffer.length ? void 0 : lastEventInBuffer?.event.eventId);
+    } finally {
+      this.notifyBufferEmpty();
     }
   }
   async reOpen(eventIds, deliveryId) {
+    const eventIdSet = new Set(eventIds);
     let lastEvent;
     const failedEvents = [];
     const reopenedEvents = [];
     this.buffer = this.buffer.filter((event) => {
-      if (event.deliveryId !== deliveryId || !eventIds.includes(event.event.eventId)) {
+      if (event.deliveryId !== deliveryId || !eventIdSet.has(event.event.eventId)) {
         return true;
       }
       if (this.options.maxRedeliveryCount > -1 && event.deliveryCount > this.options.maxRedeliveryCount) {
         this.incMetricsCounter("failed", event.event.eventType, 1);
+        this.failedCount++;
         failedEvents.push(event.event);
         lastEvent = event.event;
+        this.removeFromBufferStats(event);
         return false;
       }
       event.status = "open";
       event.deliveryId = void 0;
+      this.updateReservedStats(event, -1);
       reopenedEvents.push(event.event);
       return true;
     });
     this.updateMetricsGauges();
     if (reopenedEvents.length) {
       this.logger?.info(`Reopened ${reopenedEvents.length} events`);
-      await this.waiterEvents?.();
+      this.notifyEventWaiters();
     }
     if (!failedEvents.length) {
       return;
     }
     this.logger?.info(`Failed ${failedEvents.length} events`);
-    void this.options.processor?.failedHandler?.(failedEvents);
-    void this.finallyFailedHandler?.(failedEvents);
     if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
       this.waiterBufferThreshold?.();
     }
-    await this.updateState(this.buffer.length ? void 0 : lastEvent?.eventId);
-    if (!this.buffer.length) {
-      this.waiterBufferEmpty?.();
+    try {
+      const callbackResults = await Promise.allSettled([
+        Promise.resolve().then(() => this.options.processor?.failedHandler?.(failedEvents)),
+        Promise.resolve().then(() => this.finallyFailedHandler?.(failedEvents))
+      ]);
+      const callbackFailure = callbackResults.find(
+        (result) => result.status === "rejected"
+      );
+      if (callbackFailure) {
+        throw callbackFailure.reason;
+      }
+      await this.updateState(this.buffer.length ? void 0 : lastEvent?.eventId);
+    } finally {
+      this.notifyBufferEmpty();
     }
   }
   // #endregion
   // #region Pusher
+  /**
+   * Guarantee a live process loop whenever the pump is running with a
+   * processor. Called from the fetch loop, so a delivery loop that exited for
+   * any reason — most importantly a restart, which clears `running` while the
+   * loop is mid-batch — comes back within one fetch iteration instead of
+   * leaving a pump that pulls but never delivers.
+   */
+  ensureProcessLoop() {
+    if (!this.options.processor || !this.running || this.paused || this.activeProcessLoopGeneration === this.processLoopGeneration || this.processLoopBackoffGeneration === this.processLoopGeneration) {
+      return;
+    }
+    this.startProcessLoop();
+  }
   startProcessLoop() {
-    this.processLoop().catch((error) => {
+    const generation = this.processLoopGeneration;
+    if (!this.options.processor || this.paused || !this.isCurrentProcessLoop(generation) || this.activeProcessLoopGeneration === generation || this.processLoopBackoffGeneration === generation) {
+      return;
+    }
+    this.activeProcessLoopGeneration = generation;
+    this.processLoop(generation).then(() => {
+      if (this.activeProcessLoopGeneration !== generation) return;
+      this.activeProcessLoopGeneration = void 0;
+      if (this.isCurrentProcessLoop(generation)) {
+        this.startProcessLoop();
+      }
+    }).catch((error) => {
+      if (this.activeProcessLoopGeneration !== generation) return;
+      this.activeProcessLoopGeneration = void 0;
       this.logger?.error("Error in processor", { error });
-      if (!this.running) return;
+      if (!this.isCurrentProcessLoop(generation)) return;
       this.processLoopRestartAttempts++;
       const delay = Math.min(1e3 * Math.pow(2, this.processLoopRestartAttempts - 1), 3e4);
       this.logger?.warn(`Restarting process loop in ${delay}ms (attempt ${this.processLoopRestartAttempts})`);
-      setTimeout(() => {
-        if (!this.running) return;
+      this.processLoopBackoffGeneration = generation;
+      this.processLoopRestartTimer = setTimeout(() => {
+        if (this.processLoopBackoffGeneration !== generation) return;
+        this.processLoopBackoffGeneration = void 0;
+        this.processLoopRestartTimer = void 0;
+        if (!this.isCurrentProcessLoop(generation)) return;
         this.startProcessLoop();
       }, delay);
     });
   }
-  async processLoop() {
-    while (this.running) {
+  isCurrentProcessLoop(generation) {
+    return this.running && generation === this.processLoopGeneration;
+  }
+  async processLoop(generation) {
+    while (this.isCurrentProcessLoop(generation)) {
+      if (this.paused) return;
       try {
-        const events = await this.reserve(this.options.processor?.concurrency ?? 1);
-        await this.options.processor?.handler(events);
+        const events = await this.reserveInternal(this.options.processor?.concurrency ?? 1, generation);
+        if (!this.isCurrentProcessLoop(generation)) return;
+        if (!events.length) continue;
+        await this.replayObserver.observeHandler(events, async () => {
+          await this.options.processor?.handler(events);
+        });
+        if (!this.isCurrentProcessLoop(generation)) return;
         await this.acknowledge(events.map((event) => event.eventId));
         this.processLoopRestartAttempts = 0;
       } catch (error) {
@@ -1074,54 +1442,71 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
   }
   // #endregion
   // #region Metrics
-  updateMetricsGauges() {
-    const stats = /* @__PURE__ */ new Map();
-    for (const eventType of this.dataSource.eventTypes) {
-      stats.set(eventType, {
-        eventCount: 0,
-        eventReservedCount: 0,
-        eventSizeBytes: 0
-      });
-    }
-    for (const item of this.buffer) {
-      const stat = stats.get(item.event.eventType);
-      if (!stat) {
-        continue;
-      }
-      stat.eventCount++;
-      if (item.status === "reserved") {
-        stat.eventReservedCount++;
+  addEventsToBuffer(events) {
+    for (const event of events) {
+      const item = {
+        event,
+        status: "open",
+        deliveryCount: 0,
+        payloadSizeBytes: textEncoder.encode(JSON.stringify(event.payload)).byteLength
+      };
+      this.buffer.push(item);
+      this.bufferSizeBytes += item.payloadSizeBytes;
+      const stat = this.bufferStats.get(event.eventType);
+      if (stat) {
+        stat.eventCount++;
+        stat.eventSizeBytes += item.payloadSizeBytes;
       }
-      stat.eventSizeBytes += JSON.stringify(item.event.payload).length;
     }
-    for (const [eventType, stat] of stats) {
-      metrics.bufferEventCountGauge.set(
-        {
-          tenant: this.dataSource.tenant,
-          data_core: this.dataSource.dataCore,
-          flow_type: this.dataSource.flowType,
-          event_type: eventType
-        },
-        stat.eventCount
-      );
-      metrics.bufferReservedEventCountGauge.set(
-        {
-          tenant: this.dataSource.tenant,
-          data_core: this.dataSource.dataCore,
-          flow_type: this.dataSource.flowType,
-          event_type: eventType
-        },
-        stat.eventReservedCount
-      );
-      metrics.bufferSizeBytesGauge.set(
-        {
-          tenant: this.dataSource.tenant,
-          data_core: this.dataSource.dataCore,
-          flow_type: this.dataSource.flowType,
-          event_type: eventType
-        },
-        stat.eventSizeBytes
-      );
+  }
+  updateReservedStats(item, delta) {
+    this.bufferReservedCount += delta;
+    const stat = this.bufferStats.get(item.event.eventType);
+    if (stat) stat.eventReservedCount += delta;
+  }
+  removeFromBufferStats(item) {
+    this.bufferSizeBytes -= item.payloadSizeBytes;
+    const stat = this.bufferStats.get(item.event.eventType);
+    if (stat) {
+      stat.eventCount--;
+      stat.eventSizeBytes -= item.payloadSizeBytes;
+    }
+    if (item.status === "reserved") this.updateReservedStats(item, -1);
+  }
+  resetBufferStats() {
+    this.bufferReservedCount = 0;
+    this.bufferSizeBytes = 0;
+    for (const stat of this.bufferStats.values()) {
+      stat.eventCount = 0;
+      stat.eventReservedCount = 0;
+      stat.eventSizeBytes = 0;
+    }
+  }
+  updateMetricsGauges(synchronous = false) {
+    if (synchronous) {
+      this.gaugePublicationScheduled = false;
+      this.publishMetricsGauges();
+      return;
+    }
+    if (this.gaugePublicationScheduled) return;
+    this.gaugePublicationScheduled = true;
+    queueMicrotask(() => {
+      if (!this.gaugePublicationScheduled) return;
+      this.gaugePublicationScheduled = false;
+      this.publishMetricsGauges();
+    });
+  }
+  publishMetricsGauges() {
+    for (const [eventType, stat] of this.bufferStats) {
+      const labels = {
+        tenant: this.dataSource.tenant,
+        data_core: this.dataSource.dataCore,
+        flow_type: this.dataSource.flowType,
+        event_type: eventType
+      };
+      metrics.bufferEventCountGauge.set(labels, stat.eventCount);
+      metrics.bufferReservedEventCountGauge.set(labels, stat.eventReservedCount);
+      metrics.bufferSizeBytesGauge.set(labels, stat.eventSizeBytes);
     }
   }
   incMetricsCounter(name, eventType, value) {
@@ -1163,12 +1548,27 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
   }
   // #endregion
   // #region Waiters
-  waiterEvents;
-  async waitForEvents() {
+  publicEventWaiter;
+  processEventWaiters = /* @__PURE__ */ new Map();
+  notifyEventWaiters(includePublic = true) {
+    if (includePublic) {
+      const publicWaiter = this.publicEventWaiter;
+      this.publicEventWaiter = void 0;
+      publicWaiter?.();
+    }
+    const processWaiters = [...this.processEventWaiters.values()];
+    this.processEventWaiters.clear();
+    for (const waiter of processWaiters) waiter();
+  }
+  async waitForEvents(generation) {
     const promise = new Promise((resolve) => {
-      this.waiterEvents = resolve;
+      if (generation === void 0) {
+        this.publicEventWaiter = resolve;
+      } else {
+        this.processEventWaiters.set(generation, resolve);
+      }
     });
-    await promise;
+    await this.replayObserver.observeIdle("waiting_for_events", () => promise);
   }
   waiterBufferThreshold;
   async waitForBufferThreshold() {
@@ -1178,6 +1578,11 @@ var FlowcoreDataPump = class _FlowcoreDataPump {
     await promise;
   }
   waiterBufferEmpty;
+  notifyBufferEmpty() {
+    if (!this.buffer.length) {
+      this.waiterBufferEmpty?.();
+    }
+  }
   async waitForBufferEmpty() {
     if (!this.buffer.length) {
       return;
@@ -1429,6 +1834,7 @@ var FlowcoreDataPumpCluster = class {
     this.leaseRenewIntervalMs = options.leaseRenewIntervalMs ?? DEFAULT_LEASE_RENEW_INTERVAL_MS;
     this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
     this.workerConcurrency = options.workerConcurrency ?? DEFAULT_WORKER_CONCURRENCY;
+    this.paused = options.paused ?? false;
     this.logger = options.logger;
     this.workerHandler = options.processor?.handler;
     this.workerFailedHandler = options.processor?.failedHandler;
@@ -1443,6 +1849,10 @@ var FlowcoreDataPumpCluster = class {
   logger;
   running = false;
   isLeader = false;
+  // Delivery pause held on the CLUSTER, not on the pump. A leader change destroys the
+  // pump instance and builds a new one, so a pause stored on the pump would silently
+  // evaporate on failover and delivery would resume with no operator action.
+  paused;
   pump;
   leaderConnection;
   // leader state
@@ -1546,6 +1956,33 @@ var FlowcoreDataPumpCluster = class {
     this.startHeartbeat();
     this.startElectionLoop();
   }
+  /**
+   * Whether delivery is paused across the cluster.
+   * The flag lives on the cluster, so it survives a leader change.
+   */
+  get isPaused() {
+    return this.paused;
+  }
+  /**
+   * Pause delivery. A leader applies it to its pump immediately. A follower records it,
+   * so the pause is re-applied if this instance later becomes leader.
+   *
+   * The flag is in-memory. It does NOT survive a process restart or a full rolling
+   * deploy — persist it in the coordinator or the control plane if you need that.
+   */
+  pause() {
+    if (this.paused) return;
+    this.paused = true;
+    this.pump?.pause();
+    this.logger?.info("Cluster delivery paused", { instanceId: this.instanceId, isLeader: this.isLeader });
+  }
+  /** Resume delivery from the position where {@link pause} stopped it. */
+  resume() {
+    if (!this.paused) return;
+    this.paused = false;
+    this.pump?.resume();
+    this.logger?.info("Cluster delivery resumed", { instanceId: this.instanceId, isLeader: this.isLeader });
+  }
   async stop() {
     if (!this.running) return;
     this.running = false;
@@ -1668,6 +2105,10 @@ var FlowcoreDataPumpCluster = class {
   startPumpAsLeader() {
     const pumpOptions = {
       ...this.options,
+      // Build the pump already paused. Pausing it after `start()` would let the new
+      // leader deliver events in the gap, which is exactly what the operator asked
+      // us not to do.
+      paused: this.paused,
       processor: {
         concurrency: this.workerConcurrency,
         handler: async (events) => {
@@ -1678,6 +2119,9 @@ var FlowcoreDataPumpCluster = class {
       }
     };
     this.pump = FlowcoreDataPump.create(pumpOptions, this.options.dataSourceOverride);
+    if (this.paused) {
+      this.logger?.info("Leader pump started paused", { instanceId: this.instanceId });
+    }
     this.pump.start().catch((error) => {
       this.logger?.error("Pump error in leader mode", { error });
       if (!this.running || !this.isLeader) return;
@@ -1902,11 +2346,20 @@ export {
   NatsDistributionLeader,
   NatsDistributionWorker,
   PulseEmitter,
+  REPLAY_BATCH_SIZE_BUCKETS,
+  REPLAY_DURATION_BUCKETS_SECONDS,
+  REPLAY_EVENT_COUNT_BUCKETS,
+  REPLAY_FETCH_RESULTS,
+  REPLAY_IDLE_REASONS,
+  REPLAY_STAGE_RESULTS,
+  ReplayStageObserver,
   WsConnection,
   clusterMetrics,
+  createDataPumpMetrics,
   dataPumpPromRegistry,
   deserializeMessage,
   metrics,
   noOpLogger,
+  replayBatchSizeBucket,
   serializeMessage
 };