@flowcore/data-pump 0.21.4 → 0.23.0

package/dist/mod.js

@@ -0,0 +1,2365 @@
+// src/data-pump/data-pump.ts
+import { utc } from "@date-fns/utc";
+import { TimeUuid } from "@flowcore/time-uuid";
+import { format, startOfHour } from "date-fns";
+
+// src/data-pump/data-source.ts
+import {
+  DataCoreFetchCommand,
+  EventListCommand,
+  EventTypeListCommand,
+  FlowTypeFetchCommand,
+  TenantTranslateNameToIdCommand,
+  TimeBucketListCommand
+} from "@flowcore/sdk";
+
+// src/data-pump/flowcore-client.ts
+import { FlowcoreClient } from "@flowcore/sdk";
+
+// src/data-pump/metrics.ts
+import { Counter, Gauge, Histogram, register as defaultPromRegistry, Registry } from "prom-client";
+var dataPumpPromRegistry = new Registry();
+var SOURCE_LABELS = ["tenant", "data_core", "flow_type"];
+var EVENT_SOURCE_LABELS = [...SOURCE_LABELS, "event_type"];
+var REPLAY_DURATION_BUCKETS_SECONDS = [
+  5e-3,
+  0.01,
+  0.025,
+  0.05,
+  0.1,
+  0.25,
+  0.5,
+  1,
+  2,
+  5,
+  10,
+  20,
+  30,
+  60,
+  120
+];
+var REPLAY_EVENT_COUNT_BUCKETS = [1, 10, 50, 100, 250, 500, 1e3, 2e3, 5e3, 1e4];
+var REPLAY_BATCH_SIZE_BUCKETS = ["empty", "1", "2-10", "11-100", "101-1000", "1001+"];
+var REPLAY_FETCH_RESULTS = ["success", "no_events", "error"];
+var REPLAY_STAGE_RESULTS = ["success", "error"];
+var REPLAY_IDLE_REASONS = ["no_events", "buffer_full", "waiting_for_events"];
+function replayBatchSizeBucket(size) {
+  if (size <= 0) return "empty";
+  if (size === 1) return "1";
+  if (size <= 10) return "2-10";
+  if (size <= 100) return "11-100";
+  if (size <= 1e3) return "101-1000";
+  return "1001+";
+}
+function createDataPumpMetrics(registry, includeDefaultRegistry = false) {
+  const registers = includeDefaultRegistry ? [registry, defaultPromRegistry] : [registry];
+  const bufferEventCountGauge = new Gauge({
+    name: "flowcore_data_pump_buffer_events_gauge",
+    help: "The number of events in the buffer",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const bufferReservedEventCountGauge = new Gauge({
+    name: "flowcore_data_pump_buffer_reserved_events_gauge",
+    help: "The number of reserved events in the buffer",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const bufferSizeBytesGauge = new Gauge({
+    name: "flowcore_data_pump_buffer_size_bytes_gauge",
+    help: "The size of the buffer in bytes",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const eventsAcknowledgedCounter = new Counter({
+    name: "flowcore_data_pump_events_acknowledged_counter",
+    help: "The number of events acknowledged",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const eventsFailedCounter = new Counter({
+    name: "flowcore_data_pump_events_failed_counter",
+    help: "The number of events failed",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const eventsPulledSizeBytesCounter = new Counter({
+    name: "flowcore_data_pump_events_pulled_size_bytes_counter",
+    help: "The size of the events pulled in bytes",
+    labelNames: EVENT_SOURCE_LABELS,
+    registers
+  });
+  const sdkCommandsCounter = new Counter({
+    name: "flowcore_data_pump_sdk_commands_counter",
+    help: "The number of SDK commands",
+    labelNames: ["command"],
+    registers
+  });
+  const replayFetchDuration = new Histogram({
+    name: "flowcore_data_pump_replay_fetch_duration_seconds",
+    help: "Replay event fetch duration in seconds",
+    labelNames: [...SOURCE_LABELS, "result"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS],
+    registers
+  });
+  const replayFetchEvents = new Histogram({
+    name: "flowcore_data_pump_replay_fetch_events",
+    help: "Number of events returned by a replay fetch",
+    labelNames: SOURCE_LABELS,
+    buckets: [...REPLAY_EVENT_COUNT_BUCKETS],
+    registers
+  });
+  const replayHandlerDuration = new Histogram({
+    name: "flowcore_data_pump_replay_handler_duration_seconds",
+    help: "Replay handler duration in seconds",
+    labelNames: [...SOURCE_LABELS, "result", "batch_size_bucket"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS],
+    registers
+  });
+  const replayAcknowledgementDuration = new Histogram({
+    name: "flowcore_data_pump_replay_acknowledgement_duration_seconds",
+    help: "Replay acknowledgement duration in seconds, excluding checkpointing",
+    labelNames: [...SOURCE_LABELS, "result"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS],
+    registers
+  });
+  const replayCheckpointDuration = new Histogram({
+    name: "flowcore_data_pump_replay_checkpoint_duration_seconds",
+    help: "Replay state checkpoint duration in seconds",
+    labelNames: [...SOURCE_LABELS, "result"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS],
+    registers
+  });
+  const replayIdleDuration = new Histogram({
+    name: "flowcore_data_pump_replay_idle_duration_seconds",
+    help: "Time the replay loop spends idle",
+    labelNames: [...SOURCE_LABELS, "reason"],
+    buckets: [...REPLAY_DURATION_BUCKETS_SECONDS, 300],
+    registers
+  });
+  return {
+    bufferEventCountGauge,
+    bufferReservedEventCountGauge,
+    bufferSizeBytesGauge,
+    eventsAcknowledgedCounter,
+    eventsFailedCounter,
+    eventsPulledSizeBytesCounter,
+    sdkCommandsCounter,
+    replayFetchDuration,
+    replayFetchEvents,
+    replayHandlerDuration,
+    replayAcknowledgementDuration,
+    replayCheckpointDuration,
+    replayIdleDuration
+  };
+}
+var metrics = createDataPumpMetrics(dataPumpPromRegistry, true);
+var ReplayStageObserver = class {
+  constructor(stageMetrics, source, now = () => performance.now()) {
+    this.stageMetrics = stageMetrics;
+    this.source = source;
+    this.now = now;
+  }
+  stageMetrics;
+  source;
+  now;
+  async observeFetch(operation) {
+    const startedAt = this.now();
+    try {
+      const result = await operation();
+      this.stageMetrics.replayFetchDuration.observe(
+        { ...this.source, result: result.events.length ? "success" : "no_events" },
+        this.elapsedSeconds(startedAt)
+      );
+      this.stageMetrics.replayFetchEvents.observe(this.source, result.events.length);
+      return result;
+    } catch (error) {
+      this.stageMetrics.replayFetchDuration.observe({ ...this.source, result: "error" }, this.elapsedSeconds(startedAt));
+      throw error;
+    }
+  }
+  async observeHandler(events, operation) {
+    const startedAt = this.now();
+    const batchSize = replayBatchSizeBucket(events.length);
+    try {
+      const result = await operation();
+      this.stageMetrics.replayHandlerDuration.observe(
+        { ...this.source, result: "success", batch_size_bucket: batchSize },
+        this.elapsedSeconds(startedAt)
+      );
+      return result;
+    } catch (error) {
+      this.stageMetrics.replayHandlerDuration.observe(
+        { ...this.source, result: "error", batch_size_bucket: batchSize },
+        this.elapsedSeconds(startedAt)
+      );
+      throw error;
+    }
+  }
+  observeAcknowledgement(operation) {
+    const startedAt = this.now();
+    try {
+      const result = operation();
+      this.stageMetrics.replayAcknowledgementDuration.observe(
+        { ...this.source, result: "success" },
+        this.elapsedSeconds(startedAt)
+      );
+      return result;
+    } catch (error) {
+      this.stageMetrics.replayAcknowledgementDuration.observe(
+        { ...this.source, result: "error" },
+        this.elapsedSeconds(startedAt)
+      );
+      throw error;
+    }
+  }
+  observeCheckpoint(operation) {
+    return this.observeResult(this.stageMetrics.replayCheckpointDuration, operation);
+  }
+  async observeIdle(reason, operation) {
+    const startedAt = this.now();
+    try {
+      return await operation();
+    } finally {
+      this.stageMetrics.replayIdleDuration.observe({ ...this.source, reason }, this.elapsedSeconds(startedAt));
+    }
+  }
+  async observeResult(histogram, operation) {
+    const startedAt = this.now();
+    try {
+      const result = await operation();
+      histogram.observe({ ...this.source, result: "success" }, this.elapsedSeconds(startedAt));
+      return result;
+    } catch (error) {
+      histogram.observe({ ...this.source, result: "error" }, this.elapsedSeconds(startedAt));
+      throw error;
+    }
+  }
+  elapsedSeconds(startedAt) {
+    return Math.max(0, this.now() - startedAt) / 1e3;
+  }
+};
+var activeWorkersGauge = new Gauge({
+  name: "flowcore_data_pump_cluster_active_workers_gauge",
+  help: "The number of active worker connections"
+});
+var leaderStatusGauge = new Gauge({
+  name: "flowcore_data_pump_cluster_leader_status_gauge",
+  help: "Whether this instance is the leader (1) or not (0)"
+});
+var eventsDistributedCounter = new Counter({
+  name: "flowcore_data_pump_cluster_events_distributed_counter",
+  help: "The number of events distributed to workers"
+});
+var workerAcksCounter = new Counter({
+  name: "flowcore_data_pump_cluster_worker_acks_counter",
+  help: "The number of successful worker acknowledgements"
+});
+var workerFailsCounter = new Counter({
+  name: "flowcore_data_pump_cluster_worker_fails_counter",
+  help: "The number of failed worker deliveries"
+});
+dataPumpPromRegistry.registerMetric(activeWorkersGauge);
+dataPumpPromRegistry.registerMetric(leaderStatusGauge);
+dataPumpPromRegistry.registerMetric(eventsDistributedCounter);
+dataPumpPromRegistry.registerMetric(workerAcksCounter);
+dataPumpPromRegistry.registerMetric(workerFailsCounter);
+var clusterMetrics = {
+  activeWorkersGauge,
+  leaderStatusGauge,
+  eventsDistributedCounter,
+  workerAcksCounter,
+  workerFailsCounter
+};
+
+// src/data-pump/flowcore-client.ts
+var FlowcoreClientWithMetrics = class extends FlowcoreClient {
+  execute(command, directMode) {
+    metrics.sdkCommandsCounter.inc({ command: command.constructor.name }, 1);
+    return super.execute(command, directMode);
+  }
+};
+function getFlowcoreClient(auth, baseUrlOverride) {
+  const client = "apiKey" in auth ? new FlowcoreClientWithMetrics({
+    apiKey: auth.apiKey,
+    // apiKeyId is guaranteed to be set — FlowcoreDataPump.create() parses it from fc_ keys
+    apiKeyId: auth.apiKeyId
+  }) : new FlowcoreClientWithMetrics(auth);
+  if (baseUrlOverride) {
+    client.setBaseUrl(baseUrlOverride);
+  }
+  return client;
+}
+
+// src/data-pump/data-source.ts
+var FlowcoreDataSource = class {
+  /**
+   * Creates a new FlowcoreDataSource instance
+   * @param options - Configuration options for the data source
+   */
+  constructor(options) {
+    this.options = options;
+    this.flowcoreClient = getFlowcoreClient(this.options.auth, this.options.baseUrlOverride);
+  }
+  options;
+  /** Flowcore client instance */
+  flowcoreClient;
+  /** Cached tenant ID */
+  tenantId;
+  /** Cached data core ID */
+  dataCoreId;
+  /** Cached flow type ID */
+  flowTypeId;
+  /** Cached event type IDs */
+  eventTypeIds;
+  /** Cached time buckets */
+  timeBuckets;
+  /** Index for each cached bucket. */
+  timeBucketIndexes = /* @__PURE__ */ new Map();
+  /**
+   * Gets the tenant name from the data source configuration
+   * @returns Tenant name
+   */
+  get tenant() {
+    return this.options.dataSource.tenant;
+  }
+  /**
+   * Gets the data core name from the data source configuration
+   * @returns Data core name
+   */
+  get dataCore() {
+    return this.options.dataSource.dataCore;
+  }
+  /**
+   * Gets the flow type name from the data source configuration
+   * @returns Flow type name
+   */
+  get flowType() {
+    return this.options.dataSource.flowType;
+  }
+  /**
+   * Gets the event type names from the data source configuration
+   * @returns Array of event type names
+   */
+  get eventTypes() {
+    return this.options.dataSource.eventTypes;
+  }
+  /**
+   * Gets the tenant ID, translating from name if necessary
+   * @returns Promise that resolves to the tenant ID
+   */
+  async getTenantId() {
+    if (this.tenantId) {
+      return this.tenantId;
+    }
+    if (this.options.noTranslation) {
+      this.tenantId = this.options.dataSource.tenant;
+    } else {
+      const command = new TenantTranslateNameToIdCommand({
+        tenant: this.options.dataSource.tenant
+      });
+      const result = await this.flowcoreClient.execute(command, this.options.directMode);
+      this.tenantId = result.id;
+    }
+    return this.tenantId;
+  }
+  /**
+   * Gets the data core ID, translating from name if necessary
+   * @returns Promise that resolves to the data core ID
+   */
+  async getDataCoreId() {
+    if (this.dataCoreId) {
+      return this.dataCoreId;
+    }
+    if (this.options.noTranslation) {
+      this.dataCoreId = this.options.dataSource.dataCore;
+    } else {
+      const command = new DataCoreFetchCommand({
+        tenant: this.options.dataSource.tenant,
+        dataCore: this.options.dataSource.dataCore
+      });
+      const result = await this.flowcoreClient.execute(command, this.options.directMode);
+      this.dataCoreId = result.id;
+    }
+    return this.dataCoreId;
+  }
+  /**
+   * Gets the flow type ID, translating from name if necessary
+   * @returns Promise that resolves to the flow type ID
+   */
+  async getFlowTypeId() {
+    if (this.flowTypeId) {
+      return this.flowTypeId;
+    }
+    if (this.options.noTranslation) {
+      this.flowTypeId = this.options.dataSource.flowType;
+    } else {
+      const command = new FlowTypeFetchCommand({
+        dataCoreId: await this.getDataCoreId(),
+        flowType: this.options.dataSource.flowType
+      });
+      const result = await this.flowcoreClient.execute(command, this.options.directMode);
+      this.flowTypeId = result.id;
+    }
+    return this.flowTypeId;
+  }
+  /**
+   * Gets the event type IDs, translating from names if necessary
+   * @returns Promise that resolves to an array of event type IDs
+   * @throws Error if an event type is not found
+   */
+  async getEventTypeIds() {
+    if (this.eventTypeIds) {
+      return this.eventTypeIds;
+    }
+    if (this.options.noTranslation) {
+      this.eventTypeIds = this.options.dataSource.eventTypes;
+    } else {
+      const command = new EventTypeListCommand({
+        flowTypeId: await this.getFlowTypeId()
+      });
+      const results = await this.flowcoreClient.execute(command, this.options.directMode);
+      const eventTypeIds = [];
+      for (const eventType of this.eventTypes) {
+        const found = results.find((result) => result.name === eventType);
+        if (!found) {
+          throw new Error(`Event type ${eventType} not found`);
+        }
+        eventTypeIds.push(found.id);
+      }
+      this.eventTypeIds = eventTypeIds;
+    }
+    return this.eventTypeIds;
+  }
+  /**
+   * Gets all time buckets for the configured event types
+   * @param force - When true, forces a refresh of cached time buckets
+   * @returns Promise that resolves to an array of time bucket strings
+   */
+  async getTimeBuckets(force = false) {
+    if (this.timeBuckets && !force) {
+      return this.timeBuckets;
+    }
+    let cursor;
+    const timeBuckets = [];
+    do {
+      const result = await this.flowcoreClient.execute(
+        new TimeBucketListCommand({
+          tenant: this.options.dataSource.tenant,
+          eventTypeId: await this.getEventTypeIds(),
+          cursor: cursor || void 0,
+          pageSize: 1e4,
+          order: "asc"
+        }),
+        this.options.directMode
+      );
+      timeBuckets.push(...result.timeBuckets);
+      cursor = result.nextCursor;
+    } while (cursor !== void 0);
+    const normalizedTimeBuckets = [...new Set(timeBuckets.sort())];
+    this.timeBuckets = normalizedTimeBuckets;
+    this.timeBucketIndexes = /* @__PURE__ */ new Map();
+    for (let index = 0; index < normalizedTimeBuckets.length; index++) {
+      this.timeBucketIndexes.set(normalizedTimeBuckets[index], index);
+    }
+    return this.timeBuckets;
+  }
+  lowerBoundTimeBucket(timeBucket) {
+    const timeBuckets = this.timeBuckets ?? [];
+    const target = Number.parseFloat(timeBucket);
+    let low = 0;
+    let high = timeBuckets.length;
+    while (low < high) {
+      const middle = low + Math.floor((high - low) / 2);
+      if (Number.parseFloat(timeBuckets[middle]) < target) low = middle + 1;
+      else high = middle;
+    }
+    return low;
+  }
+  upperBoundTimeBucket(timeBucket) {
+    const timeBuckets = this.timeBuckets ?? [];
+    const target = Number.parseFloat(timeBucket);
+    let low = 0;
+    let high = timeBuckets.length;
+    while (low < high) {
+      const middle = low + Math.floor((high - low) / 2);
+      if (Number.parseFloat(timeBuckets[middle]) <= target) low = middle + 1;
+      else high = middle;
+    }
+    return low;
+  }
+  /**
+   * Gets the next time bucket after the specified time bucket
+   * @param timeBucket - The reference time bucket
+   * @returns Promise that resolves to the next time bucket, or null if none exists
+   */
+  async getNextTimeBucket(timeBucket) {
+    const closestTimeBucket = await this.getClosestTimeBucket(timeBucket);
+    if (!closestTimeBucket) {
+      return null;
+    }
+    const timeBuckets = await this.getTimeBuckets();
+    const index = this.timeBucketIndexes.get(closestTimeBucket);
+    if (index === void 0) {
+      throw new Error(`Could not get next timeBucket, timeBucket ${timeBucket} not found`);
+    }
+    return timeBuckets[index + 1] ?? null;
+  }
+  /**
+   * Gets the closest time bucket to the specified time bucket
+   * @param timeBucket - The reference time bucket
+   * @param getBefore - When true, gets the closest time bucket before the reference
+   * @returns Promise that resolves to the closest time bucket, or null if none exists
+   * @throws Error if the time bucket format is invalid
+   */
+  async getClosestTimeBucket(timeBucket, getBefore = false) {
+    const timeBuckets = await this.getTimeBuckets();
+    if (!timeBucket.match(/^\d{14}$/)) {
+      throw new Error(`Invalid timebucket: ${timeBucket}`);
+    }
+    if (!timeBuckets.length) {
+      return null;
+    }
+    if (getBefore) {
+      const index = this.upperBoundTimeBucket(timeBucket) - 1;
+      return timeBuckets[index] ?? timeBuckets[timeBuckets.length - 1];
+    }
+    return timeBuckets[this.lowerBoundTimeBucket(timeBucket)] ?? timeBuckets[timeBuckets.length - 1];
+  }
+  /**
+   * Gets events from the data source starting from a specific state
+   * @param from - The state to start from (includes timeBucket and eventId)
+   * @param amount - Maximum number of events to retrieve
+   * @param toEventId - Optional ID to stop at when retrieving events
+   * @returns Promise that resolves to an array of Flowcore events
+   */
+  async getEvents(from, amount, toEventId, cursor, includeSensitiveData = false) {
+    const eventTypeIds = await this.getEventTypeIds();
+    const result = await this.flowcoreClient.execute(
+      new EventListCommand({
+        tenant: this.options.dataSource.tenant,
+        eventTypeId: eventTypeIds,
+        timeBucket: from.timeBucket,
+        afterEventId: !cursor ? from.eventId : void 0,
+        pageSize: amount,
+        toEventId,
+        cursor,
+        includeSensitiveData
+      }),
+      this.options.directMode
+    );
+    return result;
+  }
+};
+
+// src/data-pump/notifier.ts
+import { NotificationClient } from "@flowcore/sdk";
+import { Subject } from "rxjs";
+
+// src/data-pump/nats-connection.ts
+import * as Nats from "nats";
+var NatsConnectionManager = class {
+  constructor(servers, logger) {
+    this.servers = servers;
+    this.logger = logger;
+  }
+  servers;
+  logger;
+  connection;
+  connecting;
+  connect() {
+    if (this.connection && !this.connection.isClosed()) {
+      return Promise.resolve(this.connection);
+    }
+    if (this.connecting) {
+      return this.connecting;
+    }
+    this.connecting = (async () => {
+      this.logger?.debug("Connecting to NATS", { servers: this.servers });
+      const conn = await Nats.connect({ servers: this.servers });
+      this.connection = conn;
+      this.connecting = void 0;
+      this.logger?.info("Connected to NATS");
+      return conn;
+    })();
+    return this.connecting;
+  }
+  async close() {
+    if (this.connection && !this.connection.isClosed()) {
+      this.logger?.debug("Closing NATS connection");
+      await this.connection.drain();
+      this.connection = void 0;
+    }
+    this.connecting = void 0;
+  }
+  get isConnected() {
+    return !!this.connection && !this.connection.isClosed();
+  }
+};
+
+// src/data-pump/no-op-logger.ts
+var noOpLogger = {
+  debug: () => {
+  },
+  info: () => {
+  },
+  warn: () => {
+  },
+  error: console.error
+};
+
+// src/data-pump/notifier.ts
+var DEFAULT_TIMEOUT_MS = 2e4;
+var FlowcoreNotifier = class {
+  constructor(options) {
+    this.options = options;
+    this.dataSource = new FlowcoreDataSource({
+      auth: this.options.auth,
+      dataSource: this.options.dataSource,
+      directMode: this.options.directMode,
+      noTranslation: this.options.noTranslation
+    });
+    if (this.options.natsConnectionManager) {
+      this.natsManager = this.options.natsConnectionManager;
+    } else if (this.options.natsServers) {
+      this.natsManager = new NatsConnectionManager(this.options.natsServers, this.options.logger);
+    }
+  }
+  options;
+  dataSource;
+  natsManager;
+  subject;
+  notificationClient;
+  eventResolver;
+  timer;
+  wait(signal) {
+    if (this.options.natsServers) {
+      return this.waitNats();
+    } else if (this.options.pollerIntervalMs) {
+      return this.waitPoller(this.options.pollerIntervalMs, signal);
+    }
+    return this.waitWebSocket(signal);
+  }
+  async waitPoller(intervalMs, signal) {
+    this.options.logger?.debug("Waiting for poller");
+    const promise = new Promise((resolve) => {
+      this.eventResolver = resolve;
+    });
+    signal?.addEventListener("abort", () => this.eventResolver?.());
+    setTimeout(() => this.eventResolver?.(), Math.min(intervalMs, 1e3));
+    await promise;
+  }
+  async waitNats(signal) {
+    this.options.logger?.debug("Waiting for nats");
+    const nats = await this.natsManager.connect();
+    const dataCoreId = await this.dataSource.getDataCoreId();
+    const topics = this.dataSource.eventTypes.map(
+      (eventType) => `stored.event.notify.0.${dataCoreId}.${this.dataSource.flowType}.${eventType}`
+    );
+    const promise = new Promise((resolve) => {
+      this.eventResolver = resolve;
+    });
+    const subscriptions = [];
+    for (const topic of topics) {
+      subscriptions.push(
+        nats.subscribe(topic, {
+          callback: () => {
+            this.options.logger?.debug(`Received event from nats: ${topic}`);
+            this.eventResolver?.();
+          }
+        })
+      );
+    }
+    clearTimeout(this.timer);
+    this.timer = setTimeout(() => this.eventResolver?.(), this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+    signal?.addEventListener("abort", () => this.eventResolver?.());
+    await promise;
+    this.options.logger?.debug("Unsubscribing from nats");
+    for (const subscription of subscriptions) {
+      subscription.unsubscribe();
+    }
+  }
+  onWebSocketEvent(event) {
+    if (this.dataSource.eventTypes.includes(event.data.eventType)) {
+      this.eventResolver?.();
+    }
+  }
+  async waitWebSocket(signal) {
+    this.options.logger?.debug("Waiting for web socket");
+    const promise = new Promise((resolve) => {
+      this.eventResolver = resolve;
+    });
+    this.subject = new Subject();
+    this.subject.subscribe({
+      next: this.onWebSocketEvent.bind(this),
+      error: (error) => {
+        this.options.logger?.error("Notification stream error:", { error });
+        this.eventResolver?.();
+      }
+    });
+    clearTimeout(this.timer);
+    this.timer = setTimeout(() => this.eventResolver?.(), this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+    signal?.addEventListener("abort", () => this.eventResolver?.());
+    this.notificationClient = _internals.createNotificationClient(
+      this.subject,
+      this.webSocketAuth(),
+      {
+        tenant: this.dataSource.tenant,
+        dataCore: this.dataSource.dataCore,
+        flowType: this.dataSource.flowType
+      },
+      {
+        logger: createNotificationClientLogger(this.options.logger ?? noOpLogger),
+        reconnectInterval: 1e3
+      }
+    );
+    const connectPromise = this.notificationClient.connect();
+    try {
+      await Promise.race([connectPromise, promise]);
+      await promise;
+    } finally {
+      connectPromise.catch(() => {
+      });
+      clearTimeout(this.timer);
+      this.eventResolver = void 0;
+      this.notificationClient.disconnect();
+    }
+  }
+  webSocketAuth() {
+    if ("apiKey" in this.options.auth) {
+      return {
+        apiKey: this.options.auth.apiKey,
+        // apiKeyId is guaranteed to be set — FlowcoreDataPump.create() parses it from fc_ keys
+        apiKeyId: this.options.auth.apiKeyId
+      };
+    }
+    const getBearerToken = this.options.auth.getBearerToken;
+    return {
+      oidcClient: {
+        getToken: async () => ({
+          accessToken: await getBearerToken()
+        })
+      }
+    };
+  }
+};
+function createNotificationClientLogger(logger) {
+  return {
+    debug: (message, metadata) => logger.debug(message, metadata),
+    info: (message, metadata) => {
+      if (isNormalNotificationLifecycleInfo(message)) {
+        logger.debug(message, metadata);
+        return;
+      }
+      logger.info(message, metadata);
+    },
+    warn: (message, metadata) => logger.warn(message, metadata),
+    error: (message, metadata) => logger.error(message, metadata)
+  };
+}
+function isNormalNotificationLifecycleInfo(message) {
+  return message === "WebSocket connection opened." || message.startsWith("Connection closed:");
+}
+var _internals = {
+  createNotificationClient: (...args) => new NotificationClient(...args)
+};
+
+// src/data-pump/pulse.ts
+import { SendPumpPulseCommand } from "@flowcore/sdk";
+var PulseEmitter = class {
+  constructor(options, getSnapshot) {
+    this.options = options;
+    this.getSnapshot = getSnapshot;
+    this.intervalMs = options.intervalMs ?? 3e4;
+    this.logger = options.logger;
+    this.successLogLevel = options.successLogLevel ?? "debug";
+    this.failureLogLevel = options.failureLogLevel ?? "warn";
+  }
+  options;
+  getSnapshot;
+  interval = null;
+  startTimeout = null;
+  intervalMs;
+  logger;
+  successLogLevel;
+  failureLogLevel;
+  start() {
+    if (this.interval || this.startTimeout) return;
+    const initialDelay = Math.floor(Math.random() * this.intervalMs);
+    this.startTimeout = setTimeout(() => {
+      this.startTimeout = null;
+      this.emitSafe();
+      this.interval = setInterval(() => this.emitSafe(), this.intervalMs);
+    }, initialDelay);
+  }
+  stop() {
+    if (this.startTimeout) {
+      clearTimeout(this.startTimeout);
+      this.startTimeout = null;
+    }
+    if (this.interval) {
+      clearInterval(this.interval);
+      this.interval = null;
+    }
+  }
+  emitSafe() {
+    this.emit().catch((err) => {
+      const snapshot = this.getSnapshot();
+      const msg = err instanceof Error ? err.message : String(err);
+      this.logger?.[this.failureLogLevel]?.("Pulse emission failed", {
+        error: msg,
+        url: this.options.url,
+        pathwayId: snapshot?.pathwayId,
+        flowType: snapshot?.flowType
+      });
+    });
+  }
+  async emit() {
+    const snapshot = this.getSnapshot();
+    if (!snapshot) return;
+    const client = getFlowcoreClient(this.options.auth, this.options.url);
+    const input = {
+      pathwayId: snapshot.pathwayId,
+      sourceId: snapshot.sourceId,
+      flowType: snapshot.flowType,
+      timeBucket: snapshot.timeBucket,
+      eventId: snapshot.eventId ?? null,
+      isLive: snapshot.isLive,
+      // Same widening as `sourceId` above: the CP accepts `paused`, the pinned sdk
+      // type does not declare it yet. The cast below absorbs it.
+      paused: snapshot.paused,
+      buffer: {
+        depth: snapshot.bufferDepth,
+        reserved: snapshot.bufferReserved,
+        sizeBytes: snapshot.bufferSizeBytes
+      },
+      counters: {
+        acknowledged: snapshot.acknowledgedTotal,
+        failed: snapshot.failedTotal,
+        pulled: snapshot.pulledTotal
+      },
+      uptimeMs: snapshot.uptimeMs
+    };
+    await client.execute(new SendPumpPulseCommand(input));
+    this.logger?.[this.successLogLevel]?.("Pulse sent", {
+      pathwayId: snapshot.pathwayId,
+      flowType: snapshot.flowType,
+      timeBucket: snapshot.timeBucket,
+      isLive: snapshot.isLive,
+      bufferDepth: snapshot.bufferDepth
+    });
+  }
+};
+
+// src/data-pump/data-pump.ts
+var textEncoder = new TextEncoder();
+var FlowcoreDataPump = class _FlowcoreDataPump {
+  constructor(dataSource, notifier, stateManager, options, logger) {
+    this.dataSource = dataSource;
+    this.notifier = notifier;
+    this.stateManager = stateManager;
+    this.options = options;
+    this.logger = logger;
+    this.replayObserver = new ReplayStageObserver(metrics, {
+      tenant: this.dataSource.tenant,
+      data_core: this.dataSource.dataCore,
+      flow_type: this.dataSource.flowType
+    });
+    for (const eventType of this.dataSource.eventTypes) {
+      this.bufferStats.set(eventType, { eventCount: 0, eventReservedCount: 0, eventSizeBytes: 0 });
+    }
+    this.bufferState = {
+      timeBucket: format(startOfHour(utc(/* @__PURE__ */ new Date())), "yyyyMMddHH0000"),
+      eventId: TimeUuid.now().toString()
+    };
+  }
+  dataSource;
+  notifier;
+  stateManager;
+  options;
+  logger;
+  nextCursor;
+  running = false;
+  // Delivery pause. Independent of `running`: a paused pump keeps fetching, keeps its
+  // buffer, keeps its cursor and keeps emitting pulses — it only stops handing events
+  // to the processor. The flag is sticky across `restart()` and `stop()`/`start()`, so
+  // a repositioned or bounced pump stays paused until `resume()` is called.
+  paused = false;
+  restartTo;
+  // Invalidates delivery work that crossed a stop/restart boundary. Event IDs
+  // can reappear during replay, so `running` alone cannot identify the owner.
+  processLoopGeneration = 0;
+  activeProcessLoopGeneration;
+  processLoopBackoffGeneration;
+  processLoopRestartTimer;
+  abortController;
+  buffer = [];
+  bufferState;
+  stopAtState;
+  isLive = false;
+  finallyFailedHandler;
+  pulseEmitter;
+  startedAt = 0;
+  acknowledgedCount = 0;
+  failedCount = 0;
+  pulledCount = 0;
+  processLoopRestartAttempts = 0;
+  mainLoopRestartAttempts = 0;
+  replayObserver;
+  bufferStats = /* @__PURE__ */ new Map();
+  bufferReservedCount = 0;
+  bufferSizeBytes = 0;
+  gaugePublicationScheduled = false;
+  /**
+   * Whether delivery to the processor is currently paused.
+   * A paused pump is still running: it fetches, buffers and pulses.
+   */
+  get isPaused() {
+    return this.paused;
+  }
+  get isRunning() {
+    return this.running;
+  }
+  getSnapshot() {
+    if (!this.running) return null;
+    return {
+      pathwayId: this.pulseEmitter ? "" : "",
+      // set by caller
+      flowType: this.dataSource.flowType,
+      timeBucket: this.bufferState.timeBucket,
+      eventId: this.bufferState.eventId,
+      isLive: this.isLive,
+      paused: this.paused,
+      bufferDepth: this.buffer.length,
+      bufferReserved: this.bufferReservedCount,
+      bufferSizeBytes: this.bufferSizeBytes,
+      acknowledgedTotal: this.acknowledgedCount,
+      failedTotal: this.failedCount,
+      pulledTotal: this.pulledCount,
+      uptimeMs: this.startedAt ? Date.now() - this.startedAt : 0
+    };
+  }
+  static create(options, dataSourceOverride) {
+    if ("apiKey" in options.auth && !options.auth.apiKeyId) {
+      const parts = options.auth.apiKey.split("_");
+      if (parts.length !== 3 || parts[0] !== "fc") {
+        throw new Error("Invalid API key");
+      }
+      options.auth.apiKeyId = parts[1];
+    }
+    const dataSource = dataSourceOverride ?? new FlowcoreDataSource({
+      auth: options.auth,
+      dataSource: options.dataSource,
+      baseUrlOverride: options.baseUrlOverride,
+      noTranslation: options.noTranslation,
+      directMode: options.directMode
+    });
+    const notifier = new FlowcoreNotifier({
+      auth: options.auth,
+      dataSource: options.dataSource,
+      natsServers: options.notifier?.type === "nats" ? options.notifier.servers : void 0,
+      pollerIntervalMs: options.notifier?.type === "poller" ? options.notifier.intervalMs : void 0,
+      logger: options.logger,
+      directMode: options.directMode,
+      noTranslation: options.noTranslation
+    });
+    const pump = new _FlowcoreDataPump(
+      dataSource,
+      notifier,
+      options.stateManager,
+      {
+        bufferSize: options.bufferSize ?? 1e3,
+        bufferThreshold: Math.ceil((options.bufferSize ?? 1e3) * 0.1),
+        maxRedeliveryCount: options.maxRedeliveryCount ?? 3,
+        achknowledgeTimeoutMs: options.achknowledgeTimeoutMs ?? 5e3,
+        includeSensitiveData: options.includeSensitiveData ?? false,
+        processor: options.processor,
+        stopAt: options.stopAt
+      },
+      options.logger
+    );
+    if (options.paused && options.processor) {
+      pump.paused = true;
+    }
+    if (options.pulse) {
+      const pathwayId = options.pulse.pathwayId;
+      const sourceId = options.pulse.sourceId;
+      pump.pulseEmitter = new PulseEmitter(
+        {
+          url: options.pulse.url,
+          intervalMs: options.pulse.intervalMs,
+          auth: options.auth,
+          logger: options.logger,
+          successLogLevel: options.pulse.successLogLevel,
+          failureLogLevel: options.pulse.failureLogLevel
+        },
+        () => {
+          const snapshot = pump.getSnapshot();
+          if (snapshot) {
+            snapshot.pathwayId = pathwayId;
+            snapshot.sourceId = sourceId;
+          }
+          return snapshot;
+        }
+      );
+    }
+    return pump;
+  }
+  async start(callback) {
+    this.isLive = false;
+    if (this.running) {
+      throw new Error("Data pump already running");
+    }
+    this.running = true;
+    this.startedAt = Date.now();
+    this.nextCursor = void 0;
+    this.updateMetricsGauges(true);
+    this.pulseEmitter?.start();
+    const currentState = await this.stateManager.getState();
+    const timeBucket = currentState ? await this.dataSource.getClosestTimeBucket(currentState.timeBucket) : format(startOfHour(utc(/* @__PURE__ */ new Date())), "yyyyMMddHH0000");
+    this.bufferState = {
+      timeBucket: timeBucket ?? format(startOfHour(utc(/* @__PURE__ */ new Date())), "yyyyMMddHH0000"),
+      eventId: currentState ? currentState.eventId : TimeUuid.now().toString()
+    };
+    if (this.options.stopAt) {
+      this.stopAtState = {
+        timeBucket: await this.dataSource.getClosestTimeBucket(
+          format(startOfHour(utc(this.options.stopAt)), "yyyyMMddHH0000"),
+          true
+        ) ?? format(startOfHour(utc(/* @__PURE__ */ new Date())), "yyyyMMddHH0000"),
+        eventId: TimeUuid.fromDate(this.options.stopAt).toString()
+      };
+    }
+    if (this.options.processor) {
+      this.startProcessLoop();
+    }
+    if (!callback) {
+      return this.loop();
+    }
+    this.startMainLoop(callback);
+  }
+  startMainLoop(callback) {
+    this.loop().then(() => {
+      this.mainLoopRestartAttempts = 0;
+      callback?.();
+    }).catch((error) => {
+      this.logger?.error("Error in fetch loop", { error });
+      if (!this.running) {
+        callback?.(error);
+        return;
+      }
+      this.mainLoopRestartAttempts++;
+      const delay = Math.min(1e3 * Math.pow(2, this.mainLoopRestartAttempts - 1), 3e4);
+      this.logger?.warn(`Restarting fetch loop in ${delay}ms (attempt ${this.mainLoopRestartAttempts})`);
+      setTimeout(() => {
+        if (!this.running) {
+          callback?.(error);
+          return;
+        }
+        this.startMainLoop(callback);
+      }, delay);
+    });
+  }
+  restart(state, stopAt) {
+    if (typeof state.timeBucket !== "string" || !state.timeBucket.match(/^\d{14}$/)) {
+      throw new Error(`Invalid timebucket: ${state.timeBucket}`);
+    }
+    this.restartTo = state;
+    if (stopAt !== void 0) {
+      this.options.stopAt = stopAt ?? void 0;
+    }
+    this.isLive = false;
+    this.stop(true);
+  }
+  /**
+   * Pause delivery to the processor.
+   *
+   * The fetch loop keeps running and tops the buffer up to `bufferSize`, then blocks on
+   * normal backpressure. The buffer, the cursor and the pulse emitter are untouched, so
+   * the control plane still sees a live pump. An in-flight batch finishes its handler
+   * and acknowledges, so the checkpoint stays accurate and nothing is redelivered
+   * needlessly.
+   *
+   * Idempotent. A paused pump holds up to `bufferSize` events in memory.
+   */
+  pause() {
+    if (!this.options.processor) {
+      this.logger?.warn("pause() ignored: this pump has no processor. Stop calling reserve() instead.");
+      return;
+    }
+    if (this.paused) return;
+    this.paused = true;
+    this.logger?.info("Data pump paused");
+    this.notifyEventWaiters(false);
+  }
+  /**
+   * Resume delivery to the processor from the exact position where {@link pause} stopped it.
+   * Idempotent.
+   */
+  resume() {
+    if (!this.paused) return;
+    this.paused = false;
+    if (this.processLoopRestartTimer) {
+      clearTimeout(this.processLoopRestartTimer);
+      this.processLoopRestartTimer = void 0;
+    }
+    this.processLoopBackoffGeneration = void 0;
+    this.processLoopRestartAttempts = 0;
+    this.logger?.info("Data pump resumed");
+    this.ensureProcessLoop();
+  }
+  stop(isRestart = false) {
+    this.running = false;
+    this.processLoopGeneration++;
+    this.activeProcessLoopGeneration = void 0;
+    this.processLoopBackoffGeneration = void 0;
+    if (this.processLoopRestartTimer) {
+      clearTimeout(this.processLoopRestartTimer);
+      this.processLoopRestartTimer = void 0;
+    }
+    this.processLoopRestartAttempts = 0;
+    this.mainLoopRestartAttempts = 0;
+    this.buffer = [];
+    this.resetBufferStats();
+    this.updateMetricsGauges(true);
+    this.pulseEmitter?.stop();
+    this.abortController?.abort();
+    this.waiterBufferThreshold?.();
+    this.notifyEventWaiters(!isRestart);
+  }
+  updateState(eventId) {
+    const stateManager = this.stateManager;
+    if (!stateManager.setState) {
+      return;
+    }
+    const stateEventId = eventId ?? this.buffer[0]?.event.eventId;
+    if (!stateEventId) {
+      return;
+    }
+    const date = TimeUuid.fromString(stateEventId).getDate();
+    const timeBucket = format(startOfHour(utc(date)), "yyyyMMddHH0000");
+    return this.replayObserver.observeCheckpoint(() => stateManager.setState({ timeBucket, eventId: stateEventId }));
+  }
+  async loop() {
+    do {
+      this.ensureProcessLoop();
+      const amountToFetch = this.options.bufferSize - this.buffer.length;
+      if (amountToFetch <= 0) {
+        this.logger?.info("Buffer is full, waiting for space");
+        await this.replayObserver.observeIdle("buffer_full", () => this.waitForBufferThreshold());
+        continue;
+      }
+      this.logger?.debug(`fetching ${amountToFetch} events from ${this.bufferState.timeBucket}(${this.nextCursor})`);
+      const { events, nextCursor } = await this.replayObserver.observeFetch(
+        () => this.dataSource.getEvents(
+          this.bufferState,
+          amountToFetch,
+          this.stopAtState?.eventId,
+          this.nextCursor,
+          this.options.includeSensitiveData
+        )
+      );
+      if (!this.running) {
+        break;
+      }
+      this.logger?.debug(`fetched ${events.length} events`);
+      this.pulledCount += events.length;
+      this.mainLoopRestartAttempts = 0;
+      this.addEventsToBuffer(events);
+      this.nextCursor = nextCursor;
+      this.updateMetricsGauges();
+      events.length && this.notifyEventWaiters();
+      this.bufferState.eventId = events[events.length - 1]?.eventId ?? this.bufferState.eventId;
+      if (this.stopAtState?.timeBucket && this.bufferState.timeBucket >= this.stopAtState.timeBucket && !events.length) {
+        this.logger?.info("Stopping at stopAt state");
+        await this.waitForBufferEmpty();
+        this.stop();
+        break;
+      }
+      if (!this.nextCursor) {
+        const timeBucket = await this.dataSource.getNextTimeBucket(this.bufferState.timeBucket);
+        if (!this.running) {
+          break;
+        }
+        if (timeBucket) {
+          this.bufferState.timeBucket = timeBucket;
+        } else {
+          const previousTimeBucket = this.bufferState.timeBucket;
+          this.bufferState.timeBucket = format(startOfHour(utc(/* @__PURE__ */ new Date())), "yyyyMMddHH0000");
+          if (previousTimeBucket === this.bufferState.timeBucket && !events.length) {
+            this.isLive = true;
+            this.logger?.debug("Going live...");
+            this.abortController = new AbortController();
+            await this.replayObserver.observeIdle("no_events", () => this.notifier.wait(this.abortController.signal));
+          } else if (this.isLive) {
+            await this.replayObserver.observeIdle(
+              "no_events",
+              () => new Promise((resolve) => setTimeout(resolve, 1e3))
+            );
+          }
+        }
+      }
+    } while (this.running);
+    if (this.restartTo) {
+      try {
+        await this.dataSource.getTimeBuckets(true);
+        this.restartTo.timeBucket = await this.dataSource.getClosestTimeBucket(this.restartTo.timeBucket) ?? format(startOfHour(utc(/* @__PURE__ */ new Date())), "yyyyMMddHH0000");
+        this.nextCursor = void 0;
+        this.bufferState = this.restartTo;
+        this.restartTo = void 0;
+        this.running = true;
+        this.ensureProcessLoop();
+        this.pulseEmitter?.start();
+        return this.loop();
+      } catch (error) {
+        this.logger?.error("Failed to consume restartTo, dropping it", { error });
+        this.restartTo = void 0;
+        this.notifyEventWaiters();
+        return;
+      }
+    }
+    this.logger?.info("Data pump stopped");
+  }
+  // #region Puller
+  reserve(amount) {
+    return this.reserveInternal(amount);
+  }
+  async reserveInternal(amount, generation) {
+    if (!this.running || generation !== void 0 && generation !== this.processLoopGeneration) {
+      return [];
+    }
+    if (generation !== void 0 && this.paused) {
+      return [];
+    }
+    const events = [];
+    const deliveryId = crypto.randomUUID();
+    for (const event of this.buffer) {
+      if (event.status === "open") {
+        event.status = "reserved";
+        event.deliveryId = deliveryId;
+        event.deliveryCount++;
+        this.updateReservedStats(event, 1);
+        events.push(event.event);
+        event.eventSizeBytes ??= textEncoder.encode(JSON.stringify(event.event)).byteLength;
+        this.incMetricsCounter("pulled", event.event.eventType, event.eventSizeBytes);
+        if (events.length === amount) {
+          break;
+        }
+      }
+    }
+    if (!events.length) {
+      await this.waitForEvents(generation);
+      return this.reserveInternal(amount, generation);
+    }
+    this.updateMetricsGauges();
+    setTimeout(() => {
+      void this.reOpen(
+        events.map((event) => event.eventId),
+        deliveryId
+      ).catch((error) => {
+        this.logger?.error("Failed to reopen events after acknowledgement timeout", { error });
+      });
+    }, this.options.achknowledgeTimeoutMs);
+    return events;
+  }
+  onFinalyFailed(handler) {
+    this.finallyFailedHandler = handler;
+  }
+  async acknowledge(eventIds) {
+    if (!this.running) {
+      return;
+    }
+    const eventIdSet = new Set(eventIds);
+    const checkpointEventId = this.replayObserver.observeAcknowledgement(() => {
+      const lastEventInBuffer = this.buffer[this.buffer.length - 1];
+      this.buffer = this.buffer.filter((event) => {
+        if (eventIdSet.has(event.event.eventId)) {
+          this.incMetricsCounter("acknowledged", event.event.eventType, 1);
+          this.acknowledgedCount++;
+          this.removeFromBufferStats(event);
+          return false;
+        }
+        return true;
+      });
+      if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+        this.waiterBufferThreshold?.();
+      }
+      return this.buffer.length ? void 0 : lastEventInBuffer?.event.eventId;
+    });
+    this.updateMetricsGauges();
+    try {
+      await this.updateState(checkpointEventId);
+    } finally {
+      this.notifyBufferEmpty();
+    }
+  }
+  async fail(eventIds) {
+    if (!this.running || !eventIds.length) {
+      return;
+    }
+    const lastEventInBuffer = this.buffer[this.buffer.length - 1];
+    const eventIdSet = new Set(eventIds);
+    const failedEvents = [];
+    this.buffer = this.buffer.filter((event) => {
+      if (eventIdSet.has(event.event.eventId)) {
+        this.incMetricsCounter("failed", event.event.eventType, 1);
+        this.failedCount++;
+        failedEvents.push(event.event);
+        this.removeFromBufferStats(event);
+        return false;
+      }
+      return true;
+    });
+    this.logger?.info(`Failed ${failedEvents.length} events`);
+    if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+      this.waiterBufferThreshold?.();
+    }
+    this.updateMetricsGauges();
+    try {
+      await this.options.processor?.failedHandler?.(failedEvents);
+      await this.updateState(this.buffer.length ? void 0 : lastEventInBuffer?.event.eventId);
+    } finally {
+      this.notifyBufferEmpty();
+    }
+  }
+  async reOpen(eventIds, deliveryId) {
+    const eventIdSet = new Set(eventIds);
+    let lastEvent;
+    const failedEvents = [];
+    const reopenedEvents = [];
+    this.buffer = this.buffer.filter((event) => {
+      if (event.deliveryId !== deliveryId || !eventIdSet.has(event.event.eventId)) {
+        return true;
+      }
+      if (this.options.maxRedeliveryCount > -1 && event.deliveryCount > this.options.maxRedeliveryCount) {
+        this.incMetricsCounter("failed", event.event.eventType, 1);
+        this.failedCount++;
+        failedEvents.push(event.event);
+        lastEvent = event.event;
+        this.removeFromBufferStats(event);
+        return false;
+      }
+      event.status = "open";
+      event.deliveryId = void 0;
+      this.updateReservedStats(event, -1);
+      reopenedEvents.push(event.event);
+      return true;
+    });
+    this.updateMetricsGauges();
+    if (reopenedEvents.length) {
+      this.logger?.info(`Reopened ${reopenedEvents.length} events`);
+      this.notifyEventWaiters();
+    }
+    if (!failedEvents.length) {
+      return;
+    }
+    this.logger?.info(`Failed ${failedEvents.length} events`);
+    if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+      this.waiterBufferThreshold?.();
+    }
+    try {
+      const callbackResults = await Promise.allSettled([
+        Promise.resolve().then(() => this.options.processor?.failedHandler?.(failedEvents)),
+        Promise.resolve().then(() => this.finallyFailedHandler?.(failedEvents))
+      ]);
+      const callbackFailure = callbackResults.find(
+        (result) => result.status === "rejected"
+      );
+      if (callbackFailure) {
+        throw callbackFailure.reason;
+      }
+      await this.updateState(this.buffer.length ? void 0 : lastEvent?.eventId);
+    } finally {
+      this.notifyBufferEmpty();
+    }
+  }
+  // #endregion
+  // #region Pusher
+  /**
+   * Guarantee a live process loop whenever the pump is running with a
+   * processor. Called from the fetch loop, so a delivery loop that exited for
+   * any reason — most importantly a restart, which clears `running` while the
+   * loop is mid-batch — comes back within one fetch iteration instead of
+   * leaving a pump that pulls but never delivers.
+   */
+  ensureProcessLoop() {
+    if (!this.options.processor || !this.running || this.paused || this.activeProcessLoopGeneration === this.processLoopGeneration || this.processLoopBackoffGeneration === this.processLoopGeneration) {
+      return;
+    }
+    this.startProcessLoop();
+  }
+  startProcessLoop() {
+    const generation = this.processLoopGeneration;
+    if (!this.options.processor || this.paused || !this.isCurrentProcessLoop(generation) || this.activeProcessLoopGeneration === generation || this.processLoopBackoffGeneration === generation) {
+      return;
+    }
+    this.activeProcessLoopGeneration = generation;
+    this.processLoop(generation).then(() => {
+      if (this.activeProcessLoopGeneration !== generation) return;
+      this.activeProcessLoopGeneration = void 0;
+      if (this.isCurrentProcessLoop(generation)) {
+        this.startProcessLoop();
+      }
+    }).catch((error) => {
+      if (this.activeProcessLoopGeneration !== generation) return;
+      this.activeProcessLoopGeneration = void 0;
+      this.logger?.error("Error in processor", { error });
+      if (!this.isCurrentProcessLoop(generation)) return;
+      this.processLoopRestartAttempts++;
+      const delay = Math.min(1e3 * Math.pow(2, this.processLoopRestartAttempts - 1), 3e4);
+      this.logger?.warn(`Restarting process loop in ${delay}ms (attempt ${this.processLoopRestartAttempts})`);
+      this.processLoopBackoffGeneration = generation;
+      this.processLoopRestartTimer = setTimeout(() => {
+        if (this.processLoopBackoffGeneration !== generation) return;
+        this.processLoopBackoffGeneration = void 0;
+        this.processLoopRestartTimer = void 0;
+        if (!this.isCurrentProcessLoop(generation)) return;
+        this.startProcessLoop();
+      }, delay);
+    });
+  }
+  isCurrentProcessLoop(generation) {
+    return this.running && generation === this.processLoopGeneration;
+  }
+  async processLoop(generation) {
+    while (this.isCurrentProcessLoop(generation)) {
+      if (this.paused) return;
+      try {
+        const events = await this.reserveInternal(this.options.processor?.concurrency ?? 1, generation);
+        if (!this.isCurrentProcessLoop(generation)) return;
+        if (!events.length) continue;
+        await this.replayObserver.observeHandler(events, async () => {
+          await this.options.processor?.handler(events);
+        });
+        if (!this.isCurrentProcessLoop(generation)) return;
+        await this.acknowledge(events.map((event) => event.eventId));
+        this.processLoopRestartAttempts = 0;
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        this.logger?.error(`Failed to process events: ${errorMessage}`);
+      }
+    }
+  }
+  // #endregion
+  // #region Metrics
+  addEventsToBuffer(events) {
+    for (const event of events) {
+      const item = {
+        event,
+        status: "open",
+        deliveryCount: 0,
+        payloadSizeBytes: textEncoder.encode(JSON.stringify(event.payload)).byteLength
+      };
+      this.buffer.push(item);
+      this.bufferSizeBytes += item.payloadSizeBytes;
+      const stat = this.bufferStats.get(event.eventType);
+      if (stat) {
+        stat.eventCount++;
+        stat.eventSizeBytes += item.payloadSizeBytes;
+      }
+    }
+  }
+  updateReservedStats(item, delta) {
+    this.bufferReservedCount += delta;
+    const stat = this.bufferStats.get(item.event.eventType);
+    if (stat) stat.eventReservedCount += delta;
+  }
+  removeFromBufferStats(item) {
+    this.bufferSizeBytes -= item.payloadSizeBytes;
+    const stat = this.bufferStats.get(item.event.eventType);
+    if (stat) {
+      stat.eventCount--;
+      stat.eventSizeBytes -= item.payloadSizeBytes;
+    }
+    if (item.status === "reserved") this.updateReservedStats(item, -1);
+  }
+  resetBufferStats() {
+    this.bufferReservedCount = 0;
+    this.bufferSizeBytes = 0;
+    for (const stat of this.bufferStats.values()) {
+      stat.eventCount = 0;
+      stat.eventReservedCount = 0;
+      stat.eventSizeBytes = 0;
+    }
+  }
+  updateMetricsGauges(synchronous = false) {
+    if (synchronous) {
+      this.gaugePublicationScheduled = false;
+      this.publishMetricsGauges();
+      return;
+    }
+    if (this.gaugePublicationScheduled) return;
+    this.gaugePublicationScheduled = true;
+    queueMicrotask(() => {
+      if (!this.gaugePublicationScheduled) return;
+      this.gaugePublicationScheduled = false;
+      this.publishMetricsGauges();
+    });
+  }
+  publishMetricsGauges() {
+    for (const [eventType, stat] of this.bufferStats) {
+      const labels = {
+        tenant: this.dataSource.tenant,
+        data_core: this.dataSource.dataCore,
+        flow_type: this.dataSource.flowType,
+        event_type: eventType
+      };
+      metrics.bufferEventCountGauge.set(labels, stat.eventCount);
+      metrics.bufferReservedEventCountGauge.set(labels, stat.eventReservedCount);
+      metrics.bufferSizeBytesGauge.set(labels, stat.eventSizeBytes);
+    }
+  }
+  incMetricsCounter(name, eventType, value) {
+    switch (name) {
+      case "acknowledged":
+        metrics.eventsAcknowledgedCounter.inc(
+          {
+            tenant: this.dataSource.tenant,
+            data_core: this.dataSource.dataCore,
+            flow_type: this.dataSource.flowType,
+            event_type: eventType
+          },
+          value
+        );
+        break;
+      case "failed":
+        metrics.eventsFailedCounter.inc(
+          {
+            tenant: this.dataSource.tenant,
+            data_core: this.dataSource.dataCore,
+            flow_type: this.dataSource.flowType,
+            event_type: eventType
+          },
+          value
+        );
+        break;
+      case "pulled":
+        metrics.eventsPulledSizeBytesCounter.inc(
+          {
+            tenant: this.dataSource.tenant,
+            data_core: this.dataSource.dataCore,
+            flow_type: this.dataSource.flowType,
+            event_type: eventType
+          },
+          value
+        );
+        break;
+    }
+  }
+  // #endregion
+  // #region Waiters
+  publicEventWaiter;
+  processEventWaiters = /* @__PURE__ */ new Map();
+  notifyEventWaiters(includePublic = true) {
+    if (includePublic) {
+      const publicWaiter = this.publicEventWaiter;
+      this.publicEventWaiter = void 0;
+      publicWaiter?.();
+    }
+    const processWaiters = [...this.processEventWaiters.values()];
+    this.processEventWaiters.clear();
+    for (const waiter of processWaiters) waiter();
+  }
+  async waitForEvents(generation) {
+    const promise = new Promise((resolve) => {
+      if (generation === void 0) {
+        this.publicEventWaiter = resolve;
+      } else {
+        this.processEventWaiters.set(generation, resolve);
+      }
+    });
+    await this.replayObserver.observeIdle("waiting_for_events", () => promise);
+  }
+  waiterBufferThreshold;
+  async waitForBufferThreshold() {
+    const promise = new Promise((resolve) => {
+      this.waiterBufferThreshold = resolve;
+    });
+    await promise;
+  }
+  waiterBufferEmpty;
+  notifyBufferEmpty() {
+    if (!this.buffer.length) {
+      this.waiterBufferEmpty?.();
+    }
+  }
+  async waitForBufferEmpty() {
+    if (!this.buffer.length) {
+      return;
+    }
+    const promise = new Promise((resolve) => {
+      this.waiterBufferEmpty = resolve;
+    });
+    await promise;
+  }
+  // #endregion
+};
+
+// src/data-pump/nats-distribution.ts
+import * as Nats2 from "nats";
+var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
+var NatsDistributionLeader = class {
+  constructor(connection, subject, logger, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
+    this.connection = connection;
+    this.subject = subject;
+    this.logger = logger;
+    this.timeoutMs = timeoutMs;
+  }
+  connection;
+  subject;
+  logger;
+  timeoutMs;
+  codec = Nats2.JSONCodec();
+  async distribute(events) {
+    const deliveryId = crypto.randomUUID();
+    const request = { deliveryId, events };
+    this.logger?.debug("Distributing events via NATS", { deliveryId, count: events.length, subject: this.subject });
+    const response = await this.connection.request(this.subject, this.codec.encode(request), {
+      timeout: this.timeoutMs
+    });
+    const reply = this.codec.decode(response.data);
+    if (reply.status === "fail") {
+      throw new Error(`Worker failed to process events: ${reply.error ?? "unknown error"}`);
+    }
+  }
+};
+var NatsDistributionWorker = class {
+  constructor(connection, subject, handler, logger) {
+    this.connection = connection;
+    this.subject = subject;
+    this.handler = handler;
+    this.logger = logger;
+  }
+  connection;
+  subject;
+  handler;
+  logger;
+  subscription;
+  codec = Nats2.JSONCodec();
+  queueGroup = "data-pump-workers";
+  start() {
+    this.logger?.info("Starting NATS distribution worker", { subject: this.subject, queue: this.queueGroup });
+    this.subscription = this.connection.subscribe(this.subject, {
+      queue: this.queueGroup,
+      callback: (_err, msg) => {
+        void this.handleMessage(msg);
+      }
+    });
+  }
+  stop() {
+    if (this.subscription) {
+      this.logger?.debug("Stopping NATS distribution worker");
+      this.subscription.unsubscribe();
+      this.subscription = void 0;
+    }
+  }
+  async handleMessage(msg) {
+    try {
+      const request = this.codec.decode(msg.data);
+      this.logger?.debug("Received events from leader via NATS", {
+        deliveryId: request.deliveryId,
+        count: request.events.length
+      });
+      await this.handler(request.events);
+      const reply = { status: "ack" };
+      msg.respond(this.codec.encode(reply));
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : "unknown error";
+      this.logger?.error("Worker failed to process events via NATS", { error: errorMessage });
+      const reply = { status: "fail", error: errorMessage };
+      msg.respond(this.codec.encode(reply));
+    }
+  }
+};
+
+// src/data-pump/ws-protocol.ts
+function serializeMessage(msg) {
+  return JSON.stringify(msg);
+}
+function deserializeMessage(data) {
+  try {
+    const parsed = JSON.parse(data);
+    if (!parsed || typeof parsed !== "object" || !parsed.type) {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+var PING_INTERVAL_MS = 5e3;
+var PONG_TIMEOUT_MS = 1e4;
+var WsConnection = class {
+  constructor(ws, options) {
+    this.ws = ws;
+    this.options = options;
+    this.ws.onmessage = (event) => {
+      const msg = deserializeMessage(String(event.data));
+      if (!msg) {
+        this.options.logger?.warn("Received invalid WS message");
+        return;
+      }
+      if (msg.type === "ping") {
+        this.send({ type: "pong" });
+        return;
+      }
+      if (msg.type === "pong") {
+        this.clearPongTimeout();
+        return;
+      }
+      this.options.onMessage(msg);
+    };
+    this.ws.onclose = () => {
+      this.cleanup();
+      this.options.onClose();
+    };
+    this.ws.onerror = () => {
+      this.options.logger?.warn("WebSocket error");
+      this.close();
+    };
+    this.startPingLoop();
+  }
+  ws;
+  options;
+  pingInterval;
+  pongTimeout;
+  closed = false;
+  send(msg) {
+    if (this.closed || this.ws.readyState !== WebSocket.OPEN) {
+      return;
+    }
+    this.ws.send(serializeMessage(msg));
+  }
+  close() {
+    if (this.closed) return;
+    this.closed = true;
+    this.cleanup();
+    try {
+      this.ws.close();
+    } catch {
+    }
+  }
+  get isClosed() {
+    return this.closed;
+  }
+  startPingLoop() {
+    this.pingInterval = setInterval(() => {
+      if (this.closed) return;
+      this.send({ type: "ping" });
+      this.pongTimeout = setTimeout(() => {
+        this.options.logger?.warn("Pong timeout, closing connection");
+        this.close();
+      }, PONG_TIMEOUT_MS);
+    }, PING_INTERVAL_MS);
+  }
+  clearPongTimeout() {
+    if (this.pongTimeout) {
+      clearTimeout(this.pongTimeout);
+      this.pongTimeout = void 0;
+    }
+  }
+  cleanup() {
+    this.closed = true;
+    if (this.pingInterval) {
+      clearInterval(this.pingInterval);
+      this.pingInterval = void 0;
+    }
+    this.clearPongTimeout();
+  }
+};
+var DeliveryTracker = class {
+  pending = /* @__PURE__ */ new Map();
+  add(deliveryId, eventIds, timeoutMs) {
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this.pending.delete(deliveryId);
+        reject(new Error(`Delivery ${deliveryId} timed out`));
+      }, timeoutMs);
+      this.pending.set(deliveryId, { deliveryId, eventIds, resolve, reject, timeout });
+    });
+  }
+  ack(deliveryId) {
+    const delivery = this.pending.get(deliveryId);
+    if (!delivery) return;
+    clearTimeout(delivery.timeout);
+    this.pending.delete(deliveryId);
+    delivery.resolve();
+  }
+  fail(deliveryId) {
+    const delivery = this.pending.get(deliveryId);
+    if (!delivery) return;
+    clearTimeout(delivery.timeout);
+    this.pending.delete(deliveryId);
+    delivery.reject(new Error(`Delivery ${deliveryId} failed by worker`));
+  }
+  rejectAll(error) {
+    for (const delivery of this.pending.values()) {
+      clearTimeout(delivery.timeout);
+      delivery.reject(error);
+    }
+    this.pending.clear();
+  }
+  get size() {
+    return this.pending.size;
+  }
+  getPendingEventIds() {
+    const ids = [];
+    for (const delivery of this.pending.values()) {
+      ids.push(...delivery.eventIds);
+    }
+    return ids;
+  }
+};
+
+// src/data-pump/data-pump-cluster.ts
+var LEASE_KEY = "flowcore-data-pump-leader";
+var DISCOVERY_POLL_INTERVAL_MS = 1e4;
+var DEFAULT_LEASE_TTL_MS = 3e4;
+var DEFAULT_LEASE_RENEW_INTERVAL_MS = 1e4;
+var DEFAULT_HEARTBEAT_INTERVAL_MS = 5e3;
+var DEFAULT_STALE_THRESHOLD_MS = 3e4;
+var DEFAULT_WORKER_CONCURRENCY = 1;
+var RECONNECT_BASE_MS = 1e3;
+var RECONNECT_MAX_MS = 3e4;
+var DELIVERY_TIMEOUT_MS = 3e4;
+var FlowcoreDataPumpCluster = class {
+  constructor(options) {
+    this.options = options;
+    if (!this.useNatsDistribution && !options.advertisedAddress) {
+      throw new Error("advertisedAddress is required when not using NATS distribution (notifier.type !== 'nats')");
+    }
+    this.instanceId = crypto.randomUUID();
+    this.coordinator = options.coordinator;
+    this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
+    this.leaseRenewIntervalMs = options.leaseRenewIntervalMs ?? DEFAULT_LEASE_RENEW_INTERVAL_MS;
+    this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
+    this.workerConcurrency = options.workerConcurrency ?? DEFAULT_WORKER_CONCURRENCY;
+    this.paused = options.paused ?? false;
+    this.logger = options.logger;
+    this.workerHandler = options.processor?.handler;
+    this.workerFailedHandler = options.processor?.failedHandler;
+  }
+  options;
+  instanceId;
+  coordinator;
+  leaseTtlMs;
+  leaseRenewIntervalMs;
+  heartbeatIntervalMs;
+  workerConcurrency;
+  logger;
+  running = false;
+  isLeader = false;
+  // Delivery pause held on the CLUSTER, not on the pump. A leader change destroys the
+  // pump instance and builds a new one, so a pause stored on the pump would silently
+  // evaporate on failover and delivery would resume with no operator action.
+  paused;
+  pump;
+  leaderConnection;
+  // leader state
+  workers = /* @__PURE__ */ new Map();
+  workerRoundRobinIndex = 0;
+  leaseRenewInterval;
+  discoveryInterval;
+  // common state
+  heartbeatInterval;
+  electionInterval;
+  // worker state - handler for processing events received from leader
+  workerHandler;
+  workerFailedHandler;
+  pumpRestartAttempts = 0;
+  // NATS distribution state
+  natsConnectionManager;
+  natsDistLeader;
+  natsDistWorker;
+  get useNatsDistribution() {
+    return this.options.notifier?.type === "nats";
+  }
+  get natsDistributionSubject() {
+    const key = this.options.clusterKey ?? `${this.options.dataSource.tenant}.${this.options.dataSource.dataCore}.${this.options.dataSource.flowType}`;
+    return `data-pump.distribute.${key}`;
+  }
+  get isRunning() {
+    return this.running;
+  }
+  get isLeaderInstance() {
+    return this.isLeader;
+  }
+  get activeWorkerCount() {
+    return this.workers.size;
+  }
+  get id() {
+    return this.instanceId;
+  }
+  /**
+   * Handle an incoming WebSocket connection from a peer.
+   * Call this from your HTTP server's WebSocket upgrade handler.
+   *
+   * Example (Bun):
+   * ```typescript
+   * Bun.serve({
+   *   port: 8080,
+   *   fetch(req, server) {
+   *     if (server.upgrade(req)) return undefined
+   *     return new Response("Not found", { status: 404 })
+   *   },
+   *   websocket: {
+   *     open(socket) {
+   *       cluster.handleConnection(socket as unknown as WebSocket)
+   *     },
+   *     message() {},
+   *   },
+   * })
+   * ```
+   *
+   * Example (Node.js with ws):
+   * ```typescript
+   * import { WebSocketServer } from "ws"
+   * const wss = new WebSocketServer({ port: 8080 })
+   * wss.on("connection", (ws) => cluster.handleConnection(ws))
+   * ```
+   */
+  handleConnection(ws) {
+    this.logger?.debug("Incoming WS connection");
+    const conn = new WsConnection(ws, {
+      logger: this.logger,
+      onMessage: (msg) => this.handleWorkerMessage(msg, conn),
+      onClose: () => {
+        this.logger?.debug("Leader connection closed");
+        this.leaderConnection = void 0;
+      }
+    });
+    this.leaderConnection = conn;
+  }
+  async start() {
+    if (this.running) {
+      throw new Error("Cluster already running");
+    }
+    this.running = true;
+    this.logger?.info("Starting cluster instance", { instanceId: this.instanceId });
+    if (this.useNatsDistribution) {
+      const natsServers = this.options.notifier?.type === "nats" ? this.options.notifier.servers : [];
+      this.natsConnectionManager = new NatsConnectionManager(natsServers, this.logger);
+      const conn = await this.natsConnectionManager.connect();
+      if (this.workerHandler) {
+        this.natsDistWorker = new NatsDistributionWorker(
+          conn,
+          this.natsDistributionSubject,
+          this.workerHandler,
+          this.logger
+        );
+        this.natsDistWorker.start();
+      }
+      await this.coordinator.register(this.instanceId, "nats://distributed");
+    } else {
+      await this.coordinator.register(this.instanceId, this.options.advertisedAddress);
+    }
+    this.startHeartbeat();
+    this.startElectionLoop();
+  }
+  /**
+   * Whether delivery is paused across the cluster.
+   * The flag lives on the cluster, so it survives a leader change.
+   */
+  get isPaused() {
+    return this.paused;
+  }
+  /**
+   * Pause delivery. A leader applies it to its pump immediately. A follower records it,
+   * so the pause is re-applied if this instance later becomes leader.
+   *
+   * The flag is in-memory. It does NOT survive a process restart or a full rolling
+   * deploy — persist it in the coordinator or the control plane if you need that.
+   */
+  pause() {
+    if (this.paused) return;
+    this.paused = true;
+    this.pump?.pause();
+    this.logger?.info("Cluster delivery paused", { instanceId: this.instanceId, isLeader: this.isLeader });
+  }
+  /** Resume delivery from the position where {@link pause} stopped it. */
+  resume() {
+    if (!this.paused) return;
+    this.paused = false;
+    this.pump?.resume();
+    this.logger?.info("Cluster delivery resumed", { instanceId: this.instanceId, isLeader: this.isLeader });
+  }
+  async stop() {
+    if (!this.running) return;
+    this.running = false;
+    this.logger?.info("Stopping cluster instance", { instanceId: this.instanceId });
+    if (this.electionInterval) {
+      clearInterval(this.electionInterval);
+      this.electionInterval = void 0;
+    }
+    if (this.heartbeatInterval) {
+      clearInterval(this.heartbeatInterval);
+      this.heartbeatInterval = void 0;
+    }
+    this.stepDownAsLeader();
+    this.leaderConnection?.close();
+    this.leaderConnection = void 0;
+    this.natsDistWorker?.stop();
+    this.natsDistWorker = void 0;
+    this.natsDistLeader = void 0;
+    if (this.natsConnectionManager) {
+      await this.natsConnectionManager.close();
+      this.natsConnectionManager = void 0;
+    }
+    try {
+      await this.coordinator.releaseLease(this.instanceId, LEASE_KEY);
+    } catch {
+    }
+    try {
+      await this.coordinator.unregister(this.instanceId);
+    } catch {
+    }
+    clusterMetrics.leaderStatusGauge.set(0);
+    clusterMetrics.activeWorkersGauge.set(0);
+    this.logger?.info("Cluster instance stopped");
+  }
+  // #region Heartbeat
+  startHeartbeat() {
+    this.heartbeatInterval = setInterval(async () => {
+      if (!this.running) return;
+      try {
+        await this.coordinator.heartbeat(this.instanceId);
+      } catch (error) {
+        this.logger?.warn("Heartbeat failed", { error });
+      }
+    }, this.heartbeatIntervalMs);
+  }
+  // #endregion
+  // #region Leader Election
+  startElectionLoop() {
+    void this.attemptElection();
+    this.electionInterval = setInterval(async () => {
+      if (!this.running) return;
+      if (this.isLeader) {
+        try {
+          const renewed = await this.coordinator.renewLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+          if (!renewed) {
+            this.logger?.warn("Lease renewal failed, stepping down");
+            this.stepDownAsLeader();
+          }
+        } catch (error) {
+          this.logger?.error("Lease renewal error, stepping down", { error });
+          this.stepDownAsLeader();
+        }
+      } else {
+        await this.attemptElection();
+      }
+    }, this.leaseRenewIntervalMs);
+  }
+  async attemptElection() {
+    try {
+      const acquired = await this.coordinator.acquireLease(this.instanceId, LEASE_KEY, this.leaseTtlMs);
+      if (acquired && !this.isLeader) {
+        this.logger?.info("Won leader election", { instanceId: this.instanceId });
+        await this.becomeLeader();
+      }
+    } catch (error) {
+      this.logger?.debug("Election attempt failed", { error });
+    }
+  }
+  // #endregion
+  // #region Leader Mode
+  async becomeLeader() {
+    this.isLeader = true;
+    clusterMetrics.leaderStatusGauge.set(1);
+    this.leaderConnection?.close();
+    this.leaderConnection = void 0;
+    if (this.useNatsDistribution) {
+      const conn = await this.natsConnectionManager.connect();
+      this.natsDistLeader = new NatsDistributionLeader(conn, this.natsDistributionSubject, this.logger);
+    } else {
+      this.startWorkerDiscovery();
+      await this.discoverWorkers();
+    }
+    this.startPumpAsLeader();
+  }
+  stepDownAsLeader() {
+    if (!this.isLeader) return;
+    this.isLeader = false;
+    clusterMetrics.leaderStatusGauge.set(0);
+    if (this.discoveryInterval) {
+      clearInterval(this.discoveryInterval);
+      this.discoveryInterval = void 0;
+    }
+    if (this.leaseRenewInterval) {
+      clearInterval(this.leaseRenewInterval);
+      this.leaseRenewInterval = void 0;
+    }
+    this.natsDistLeader = void 0;
+    for (const worker of this.workers.values()) {
+      worker.deliveryTracker.rejectAll(new Error("Leader stepping down"));
+      worker.connection.close();
+    }
+    this.workers.clear();
+    clusterMetrics.activeWorkersGauge.set(0);
+    if (this.pump) {
+      this.pump.stop();
+      this.pump = void 0;
+    }
+    this.pumpRestartAttempts = 0;
+  }
+  startPumpAsLeader() {
+    const pumpOptions = {
+      ...this.options,
+      // Build the pump already paused. Pausing it after `start()` would let the new
+      // leader deliver events in the gap, which is exactly what the operator asked
+      // us not to do.
+      paused: this.paused,
+      processor: {
+        concurrency: this.workerConcurrency,
+        handler: async (events) => {
+          this.pumpRestartAttempts = 0;
+          await this.distributeEvents(events);
+        },
+        failedHandler: this.workerFailedHandler
+      }
+    };
+    this.pump = FlowcoreDataPump.create(pumpOptions, this.options.dataSourceOverride);
+    if (this.paused) {
+      this.logger?.info("Leader pump started paused", { instanceId: this.instanceId });
+    }
+    this.pump.start().catch((error) => {
+      this.logger?.error("Pump error in leader mode", { error });
+      if (!this.running || !this.isLeader) return;
+      this.pumpRestartAttempts++;
+      const delay = Math.min(RECONNECT_BASE_MS * Math.pow(2, this.pumpRestartAttempts - 1), RECONNECT_MAX_MS);
+      this.logger?.warn(`Restarting leader pump in ${delay}ms (attempt ${this.pumpRestartAttempts})`);
+      setTimeout(() => {
+        if (!this.running || !this.isLeader) return;
+        this.startPumpAsLeader();
+      }, delay);
+    });
+  }
+  distributeEvents(events) {
+    if (this.useNatsDistribution) {
+      return this.distributeEventsNats(events);
+    }
+    return this.distributeEventsWs(events);
+  }
+  async distributeEventsNats(events) {
+    if (!this.natsDistLeader) {
+      this.logger?.debug("No NATS leader distributor, processing locally");
+      if (this.workerHandler) {
+        await this.workerHandler(events);
+      }
+      return;
+    }
+    clusterMetrics.eventsDistributedCounter.inc(events.length);
+    try {
+      await this.natsDistLeader.distribute(events);
+      clusterMetrics.workerAcksCounter.inc();
+    } catch (error) {
+      clusterMetrics.workerFailsCounter.inc();
+      this.logger?.warn("NATS distribution failed, processing locally", {
+        error: error instanceof Error ? error.message : "unknown"
+      });
+      if (this.workerHandler) {
+        await this.workerHandler(events);
+      }
+    }
+  }
+  async distributeEventsWs(events) {
+    const worker = this.selectWorker();
+    if (!worker) {
+      this.logger?.debug("No workers available, processing locally");
+      if (this.workerHandler) {
+        await this.workerHandler(events);
+      }
+      return;
+    }
+    const deliveryId = crypto.randomUUID();
+    const eventIds = events.map((e) => e.eventId);
+    worker.pendingCount++;
+    clusterMetrics.eventsDistributedCounter.inc(events.length);
+    worker.connection.send({
+      type: "events",
+      deliveryId,
+      events
+    });
+    try {
+      await worker.deliveryTracker.add(deliveryId, eventIds, DELIVERY_TIMEOUT_MS);
+      worker.pendingCount--;
+      clusterMetrics.workerAcksCounter.inc();
+    } catch (error) {
+      worker.pendingCount--;
+      clusterMetrics.workerFailsCounter.inc();
+      throw error;
+    }
+  }
+  selectWorker() {
+    const available = [];
+    for (const worker of this.workers.values()) {
+      if (worker.pendingCount < this.workerConcurrency && !worker.connection.isClosed) {
+        available.push(worker);
+      }
+    }
+    if (available.length === 0) return null;
+    this.workerRoundRobinIndex = this.workerRoundRobinIndex % available.length;
+    const selected = available[this.workerRoundRobinIndex];
+    this.workerRoundRobinIndex++;
+    return selected;
+  }
+  // #endregion
+  // #region Worker Discovery
+  startWorkerDiscovery() {
+    this.discoveryInterval = setInterval(async () => {
+      if (!this.running || !this.isLeader) return;
+      await this.discoverWorkers();
+    }, DISCOVERY_POLL_INTERVAL_MS);
+  }
+  async discoverWorkers() {
+    try {
+      const instances = await this.coordinator.getInstances(DEFAULT_STALE_THRESHOLD_MS);
+      for (const instance of instances) {
+        if (instance.instanceId === this.instanceId) continue;
+        if (this.workers.has(instance.instanceId)) continue;
+        this.connectToWorker(instance.instanceId, instance.address);
+      }
+      const registeredIds = new Set(instances.map((i) => i.instanceId));
+      for (const [id, worker] of this.workers) {
+        if (!registeredIds.has(id)) {
+          this.logger?.info("Worker no longer registered, disconnecting", { instanceId: id });
+          worker.deliveryTracker.rejectAll(new Error("Worker unregistered"));
+          worker.connection.close();
+          this.workers.delete(id);
+        }
+      }
+      clusterMetrics.activeWorkersGauge.set(this.workers.size);
+    } catch (error) {
+      this.logger?.warn("Worker discovery failed", { error });
+    }
+  }
+  connectToWorker(instanceId, address) {
+    this.logger?.debug("Connecting to worker", { instanceId, address });
+    try {
+      const ws = new WebSocket(address);
+      const deliveryTracker = new DeliveryTracker();
+      const workerConn = {
+        instanceId,
+        connection: void 0,
+        deliveryTracker,
+        pendingCount: 0,
+        reconnectAttempts: 0
+      };
+      ws.onopen = () => {
+        const conn = new WsConnection(ws, {
+          logger: this.logger,
+          onMessage: (msg) => this.handleLeaderMessage(msg, instanceId),
+          onClose: () => this.handleWorkerDisconnect(instanceId)
+        });
+        workerConn.connection = conn;
+        workerConn.reconnectAttempts = 0;
+        this.workers.set(instanceId, workerConn);
+        clusterMetrics.activeWorkersGauge.set(this.workers.size);
+        this.logger?.debug("Connected to worker", { instanceId });
+      };
+      ws.onerror = () => {
+        this.logger?.warn("Failed to connect to worker", { instanceId, address });
+        this.scheduleReconnect(instanceId, address, (workerConn.reconnectAttempts || 0) + 1);
+      };
+    } catch (error) {
+      this.logger?.warn("Error connecting to worker", { instanceId, error });
+    }
+  }
+  handleWorkerDisconnect(instanceId) {
+    const worker = this.workers.get(instanceId);
+    if (!worker) return;
+    this.logger?.debug("Worker disconnected", { instanceId });
+    worker.deliveryTracker.rejectAll(new Error("Worker disconnected"));
+    this.workers.delete(instanceId);
+    clusterMetrics.activeWorkersGauge.set(this.workers.size);
+    if (this.running && this.isLeader) {
+      this.scheduleReconnect(instanceId, "", worker.reconnectAttempts + 1);
+    }
+  }
+  scheduleReconnect(instanceId, address, attempts) {
+    if (!this.running || !this.isLeader) return;
+    const delay = Math.min(RECONNECT_BASE_MS * Math.pow(2, attempts - 1), RECONNECT_MAX_MS);
+    this.logger?.debug("Scheduling reconnect", { instanceId, delay, attempts });
+    setTimeout(async () => {
+      if (!this.running || !this.isLeader) return;
+      let resolvedAddress = address;
+      if (!resolvedAddress) {
+        try {
+          const instances = await this.coordinator.getInstances(DEFAULT_STALE_THRESHOLD_MS);
+          const instance = instances.find((i) => i.instanceId === instanceId);
+          if (!instance) return;
+          resolvedAddress = instance.address;
+        } catch {
+          return;
+        }
+      }
+      if (!this.workers.has(instanceId)) {
+        this.connectToWorker(instanceId, resolvedAddress);
+      }
+    }, delay);
+  }
+  // #endregion
+  // #region Message Handlers
+  // leader receives ack/fail from workers
+  handleLeaderMessage(msg, workerInstanceId) {
+    const worker = this.workers.get(workerInstanceId);
+    if (!worker) return;
+    if (msg.type === "ack") {
+      worker.deliveryTracker.ack(msg.deliveryId);
+    } else if (msg.type === "fail") {
+      worker.deliveryTracker.fail(msg.deliveryId);
+    }
+  }
+  // worker receives events from leader
+  handleWorkerMessage(msg, conn) {
+    if (msg.type !== "events") return;
+    const { deliveryId, events } = msg;
+    this.logger?.debug("Received events from leader", { deliveryId, count: events.length });
+    void (async () => {
+      try {
+        if (this.workerHandler) {
+          await this.workerHandler(events);
+        }
+        conn.send({
+          type: "ack",
+          deliveryId,
+          eventIds: events.map((e) => e.eventId)
+        });
+      } catch (error) {
+        this.logger?.error("Worker failed to process events", { deliveryId, error });
+        conn.send({
+          type: "fail",
+          deliveryId,
+          eventIds: events.map((e) => e.eventId)
+        });
+      }
+    })();
+  }
+  // #endregion
+};
+export {
+  DeliveryTracker,
+  FlowcoreDataPump,
+  FlowcoreDataPumpCluster,
+  FlowcoreDataSource,
+  NatsConnectionManager,
+  NatsDistributionLeader,
+  NatsDistributionWorker,
+  PulseEmitter,
+  REPLAY_BATCH_SIZE_BUCKETS,
+  REPLAY_DURATION_BUCKETS_SECONDS,
+  REPLAY_EVENT_COUNT_BUCKETS,
+  REPLAY_FETCH_RESULTS,
+  REPLAY_IDLE_REASONS,
+  REPLAY_STAGE_RESULTS,
+  ReplayStageObserver,
+  WsConnection,
+  clusterMetrics,
+  createDataPumpMetrics,
+  dataPumpPromRegistry,
+  deserializeMessage,
+  metrics,
+  noOpLogger,
+  replayBatchSizeBucket,
+  serializeMessage
+};