@flowcore/data-pump 0.21.4 → 0.22.1

package/dist/mod.d.ts

@@ -0,0 +1,517 @@
+import { FlowcoreEvent, FlowcoreClient, EventListOutput } from '@flowcore/sdk';
+import PromClient from 'prom-client';
+import * as Nats from 'nats';
+
+type FlowcoreDataPumpAuth = {
+    apiKey: string;
+    /**
+     * The ID of the API key.
+     * @deprecated Use apiKey only instead (apiKeyId is only used for old api keys).
+     */
+    apiKeyId?: string;
+} | {
+    getBearerToken: () => Promise<string>;
+};
+interface FlowcoreDataPumpState {
+    timeBucket: string;
+    eventId?: string;
+}
+interface FlowcoreDataPumpDataSource {
+    tenant: string;
+    dataCore: string;
+    flowType: string;
+    eventTypes: string[];
+}
+interface FlowcoreDataPumpStateManager {
+    getState: () => Promise<FlowcoreDataPumpState | null> | FlowcoreDataPumpState | null;
+    setState?: (state: FlowcoreDataPumpState) => Promise<void> | void;
+}
+interface FlowcoreLogger {
+    debug: (message: string, metadata?: Record<string, unknown>) => void;
+    info: (message: string, metadata?: Record<string, unknown>) => void;
+    warn: (message: string, metadata?: Record<string, unknown>) => void;
+    error: (message: string | Error, metadata?: Record<string, unknown>) => void;
+}
+interface FlowcoreDataPumpProcessor {
+    concurrency?: number;
+    handler: (events: FlowcoreEvent[]) => Promise<void>;
+    failedHandler?: (events: FlowcoreEvent[]) => void | Promise<void>;
+}
+interface FlowcoreDataPumpCoordinator {
+    acquireLease(instanceId: string, key: string, ttlMs: number): Promise<boolean>;
+    renewLease(instanceId: string, key: string, ttlMs: number): Promise<boolean>;
+    releaseLease(instanceId: string, key: string): Promise<void>;
+    register(instanceId: string, address: string): Promise<void>;
+    heartbeat(instanceId: string): Promise<void>;
+    unregister(instanceId: string): Promise<void>;
+    getInstances(staleThresholdMs: number): Promise<Array<{
+        instanceId: string;
+        address: string;
+    }>>;
+}
+
+/**
+ * @module data-source
+ * @description Provides functionality to interact with Flowcore data sources
+ */
+
+/**
+ * Options for configuring a Flowcore data source
+ * @interface FlowcoreDataSourceOptions
+ */
+interface FlowcoreDataSourceOptions {
+    /** Authentication information for Flowcore */
+    auth: FlowcoreDataPumpAuth;
+    /** Data source configuration */
+    dataSource: FlowcoreDataPumpDataSource;
+    /** Optional base URL override for the Flowcore API */
+    baseUrlOverride?: string;
+    /** When true, disables name-to-ID translation for tenant, dataCore, flowType, and eventTypes */
+    noTranslation?: boolean;
+    /** When true, executes commands in direct mode */
+    directMode?: boolean;
+}
+/**
+ * Provides methods to interact with a Flowcore data source
+ * @class FlowcoreDataSource
+ */
+declare class FlowcoreDataSource {
+    protected readonly options: FlowcoreDataSourceOptions;
+    /** Flowcore client instance */
+    protected flowcoreClient: FlowcoreClient;
+    /** Cached tenant ID */
+    protected tenantId?: string;
+    /** Cached data core ID */
+    protected dataCoreId?: string;
+    /** Cached flow type ID */
+    protected flowTypeId?: string;
+    /** Cached event type IDs */
+    protected eventTypeIds?: string[];
+    /** Cached time buckets */
+    protected timeBuckets?: string[];
+    /**
+     * Creates a new FlowcoreDataSource instance
+     * @param options - Configuration options for the data source
+     */
+    constructor(options: FlowcoreDataSourceOptions);
+    /**
+     * Gets the tenant name from the data source configuration
+     * @returns Tenant name
+     */
+    get tenant(): string;
+    /**
+     * Gets the data core name from the data source configuration
+     * @returns Data core name
+     */
+    get dataCore(): string;
+    /**
+     * Gets the flow type name from the data source configuration
+     * @returns Flow type name
+     */
+    get flowType(): string;
+    /**
+     * Gets the event type names from the data source configuration
+     * @returns Array of event type names
+     */
+    get eventTypes(): string[];
+    /**
+     * Gets the tenant ID, translating from name if necessary
+     * @returns Promise that resolves to the tenant ID
+     */
+    getTenantId(): Promise<string>;
+    /**
+     * Gets the data core ID, translating from name if necessary
+     * @returns Promise that resolves to the data core ID
+     */
+    getDataCoreId(): Promise<string>;
+    /**
+     * Gets the flow type ID, translating from name if necessary
+     * @returns Promise that resolves to the flow type ID
+     */
+    getFlowTypeId(): Promise<string>;
+    /**
+     * Gets the event type IDs, translating from names if necessary
+     * @returns Promise that resolves to an array of event type IDs
+     * @throws Error if an event type is not found
+     */
+    getEventTypeIds(): Promise<string[]>;
+    /**
+     * Gets all time buckets for the configured event types
+     * @param force - When true, forces a refresh of cached time buckets
+     * @returns Promise that resolves to an array of time bucket strings
+     */
+    getTimeBuckets(force?: boolean): Promise<string[]>;
+    /**
+     * Gets the next time bucket after the specified time bucket
+     * @param timeBucket - The reference time bucket
+     * @returns Promise that resolves to the next time bucket, or null if none exists
+     */
+    getNextTimeBucket(timeBucket: string): Promise<string | null>;
+    /**
+     * Gets the closest time bucket to the specified time bucket
+     * @param timeBucket - The reference time bucket
+     * @param getBefore - When true, gets the closest time bucket before the reference
+     * @returns Promise that resolves to the closest time bucket, or null if none exists
+     * @throws Error if the time bucket format is invalid
+     */
+    getClosestTimeBucket(timeBucket: string, getBefore?: boolean): Promise<string | null>;
+    /**
+     * Gets events from the data source starting from a specific state
+     * @param from - The state to start from (includes timeBucket and eventId)
+     * @param amount - Maximum number of events to retrieve
+     * @param toEventId - Optional ID to stop at when retrieving events
+     * @returns Promise that resolves to an array of Flowcore events
+     */
+    getEvents(from: FlowcoreDataPumpState, amount: number, toEventId?: string, cursor?: string, includeSensitiveData?: boolean): Promise<EventListOutput>;
+}
+
+/**
+ * Log level for pulse emitter events. Corresponds to FlowcoreLogger methods.
+ */
+type PulseLogLevel = "debug" | "info" | "warn" | "error";
+interface PulseEmitterOptions {
+    url: string;
+    intervalMs?: number;
+    auth: FlowcoreDataPumpAuth;
+    logger?: FlowcoreLogger;
+    /** Log level for successful pulses. Defaults to 'debug'. */
+    successLogLevel?: PulseLogLevel;
+    /** Log level for pulse failures. Defaults to 'warn'. */
+    failureLogLevel?: PulseLogLevel;
+}
+interface PulseSnapshot {
+    pathwayId: string;
+    sourceId?: string;
+    flowType: string;
+    timeBucket: string;
+    eventId: string | undefined;
+    isLive: boolean;
+    bufferDepth: number;
+    bufferReserved: number;
+    bufferSizeBytes: number;
+    acknowledgedTotal: number;
+    failedTotal: number;
+    pulledTotal: number;
+    uptimeMs: number;
+}
+declare class PulseEmitter {
+    private readonly options;
+    private readonly getSnapshot;
+    private interval;
+    private startTimeout;
+    private readonly intervalMs;
+    private readonly logger?;
+    private readonly successLogLevel;
+    private readonly failureLogLevel;
+    constructor(options: PulseEmitterOptions, getSnapshot: () => PulseSnapshot | null);
+    start(): void;
+    stop(): void;
+    private emitSafe;
+    private emit;
+}
+
+interface FlowcoreDataPumpNotifierNatsOptions {
+    type: "nats";
+    servers: string[];
+}
+interface FlowcoreDataPumpNotifierWebsocketOptions {
+    type: "websocket";
+}
+interface FlowcoreDataPumpNotifierPollerOptions {
+    type: "poller";
+    /**
+     * The interval in milliseconds to poll the data pump state (min 1000 ms)
+     */
+    intervalMs: number;
+}
+type FlowcoreDataPumpNotifierOptions = FlowcoreDataPumpNotifierNatsOptions | FlowcoreDataPumpNotifierWebsocketOptions | FlowcoreDataPumpNotifierPollerOptions;
+interface FlowcoreDataPumpOptions {
+    auth: FlowcoreDataPumpAuth;
+    dataSource: FlowcoreDataPumpDataSource;
+    stateManager: FlowcoreDataPumpStateManager;
+    bufferSize?: number;
+    maxRedeliveryCount?: number;
+    achknowledgeTimeoutMs?: number;
+    includeSensitiveData?: boolean;
+    processor?: FlowcoreDataPumpProcessor;
+    notifier?: FlowcoreDataPumpNotifierOptions;
+    logger?: FlowcoreLogger;
+    stopAt?: Date;
+    baseUrlOverride?: string;
+    noTranslation?: boolean;
+    directMode?: boolean;
+    pulse?: {
+        url: string;
+        intervalMs?: number;
+        pathwayId: string;
+        sourceId?: string;
+        /** Log level for successful pulses. Defaults to 'debug'. */
+        successLogLevel?: "debug" | "info" | "warn" | "error";
+        /** Log level for pulse failures. Defaults to 'warn'. */
+        failureLogLevel?: "debug" | "info" | "warn" | "error";
+    };
+}
+declare class FlowcoreDataPump {
+    readonly dataSource: FlowcoreDataSource;
+    private readonly notifier;
+    private stateManager;
+    private readonly options;
+    private readonly logger?;
+    private nextCursor?;
+    private running;
+    private restartTo?;
+    private abortController?;
+    private buffer;
+    private bufferState;
+    private stopAtState?;
+    private isLive;
+    private finallyFailedHandler?;
+    private pulseEmitter?;
+    private startedAt;
+    private acknowledgedCount;
+    private failedCount;
+    private pulledCount;
+    private processLoopRestartAttempts;
+    private mainLoopRestartAttempts;
+    private constructor();
+    get isRunning(): boolean;
+    getSnapshot(): PulseSnapshot | null;
+    static create(options: FlowcoreDataPumpOptions, dataSourceOverride?: FlowcoreDataSource): FlowcoreDataPump;
+    start(callback?: (error?: Error) => void): Promise<void>;
+    private startMainLoop;
+    restart(state: FlowcoreDataPumpState, stopAt?: Date | null): void;
+    stop(isRestart?: boolean): void;
+    private updateState;
+    private loop;
+    reserve(amount: number): Promise<FlowcoreEvent[]>;
+    onFinalyFailed(handler: (events: FlowcoreEvent[]) => Promise<void> | void): void;
+    acknowledge(eventIds: string[]): Promise<void>;
+    fail(eventIds: string[]): Promise<void>;
+    private reOpen;
+    private startProcessLoop;
+    private processLoop;
+    private updateMetricsGauges;
+    private incMetricsCounter;
+    private waiterEvents?;
+    private waitForEvents;
+    private waiterBufferThreshold?;
+    private waitForBufferThreshold;
+    private waiterBufferEmpty?;
+    private waitForBufferEmpty;
+}
+
+interface FlowcoreDataPumpClusterOptions extends FlowcoreDataPumpOptions {
+    coordinator: FlowcoreDataPumpCoordinator;
+    advertisedAddress?: string;
+    dataSourceOverride?: FlowcoreDataSource;
+    leaseTtlMs?: number;
+    leaseRenewIntervalMs?: number;
+    heartbeatIntervalMs?: number;
+    workerConcurrency?: number;
+    clusterKey?: string;
+}
+declare class FlowcoreDataPumpCluster {
+    private readonly options;
+    private readonly instanceId;
+    private readonly coordinator;
+    private readonly leaseTtlMs;
+    private readonly leaseRenewIntervalMs;
+    private readonly heartbeatIntervalMs;
+    private readonly workerConcurrency;
+    private readonly logger?;
+    private running;
+    private isLeader;
+    private pump?;
+    private leaderConnection?;
+    private workers;
+    private workerRoundRobinIndex;
+    private leaseRenewInterval?;
+    private discoveryInterval?;
+    private heartbeatInterval?;
+    private electionInterval?;
+    private workerHandler?;
+    private workerFailedHandler?;
+    private pumpRestartAttempts;
+    private natsConnectionManager?;
+    private natsDistLeader?;
+    private natsDistWorker?;
+    constructor(options: FlowcoreDataPumpClusterOptions);
+    private get useNatsDistribution();
+    private get natsDistributionSubject();
+    get isRunning(): boolean;
+    get isLeaderInstance(): boolean;
+    get activeWorkerCount(): number;
+    get id(): string;
+    /**
+     * Handle an incoming WebSocket connection from a peer.
+     * Call this from your HTTP server's WebSocket upgrade handler.
+     *
+     * Example (Bun):
+     * ```typescript
+     * Bun.serve({
+     *   port: 8080,
+     *   fetch(req, server) {
+     *     if (server.upgrade(req)) return undefined
+     *     return new Response("Not found", { status: 404 })
+     *   },
+     *   websocket: {
+     *     open(socket) {
+     *       cluster.handleConnection(socket as unknown as WebSocket)
+     *     },
+     *     message() {},
+     *   },
+     * })
+     * ```
+     *
+     * Example (Node.js with ws):
+     * ```typescript
+     * import { WebSocketServer } from "ws"
+     * const wss = new WebSocketServer({ port: 8080 })
+     * wss.on("connection", (ws) => cluster.handleConnection(ws))
+     * ```
+     */
+    handleConnection(ws: WebSocket): void;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    private startHeartbeat;
+    private startElectionLoop;
+    private attemptElection;
+    private becomeLeader;
+    private stepDownAsLeader;
+    private startPumpAsLeader;
+    private distributeEvents;
+    private distributeEventsNats;
+    private distributeEventsWs;
+    private selectWorker;
+    private startWorkerDiscovery;
+    private discoverWorkers;
+    private connectToWorker;
+    private handleWorkerDisconnect;
+    private scheduleReconnect;
+    private handleLeaderMessage;
+    private handleWorkerMessage;
+}
+
+declare const dataPumpPromRegistry: PromClient.Registry<"text/plain; version=0.0.4; charset=utf-8">;
+declare const metrics: {
+    bufferEventCountGauge: PromClient.Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    bufferReservedEventCountGauge: PromClient.Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    bufferSizeBytesGauge: PromClient.Gauge<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsAcknowledgedCounter: PromClient.Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsFailedCounter: PromClient.Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    eventsPulledSizeBytesCounter: PromClient.Counter<"tenant" | "data_core" | "flow_type" | "event_type">;
+    sdkCommandsCounter: PromClient.Counter<"command">;
+};
+declare const clusterMetrics: {
+    activeWorkersGauge: PromClient.Gauge<string>;
+    leaderStatusGauge: PromClient.Gauge<string>;
+    eventsDistributedCounter: PromClient.Counter<string>;
+    workerAcksCounter: PromClient.Counter<string>;
+    workerFailsCounter: PromClient.Counter<string>;
+};
+
+declare class NatsConnectionManager {
+    private readonly servers;
+    private readonly logger?;
+    private connection?;
+    private connecting?;
+    constructor(servers: string[], logger?: FlowcoreLogger | undefined);
+    connect(): Promise<Nats.NatsConnection>;
+    close(): Promise<void>;
+    get isConnected(): boolean;
+}
+
+interface NatsDistributionRequest {
+    deliveryId: string;
+    events: FlowcoreEvent[];
+}
+interface NatsDistributionReply {
+    status: "ack" | "fail";
+    error?: string;
+}
+declare class NatsDistributionLeader {
+    private readonly connection;
+    private readonly subject;
+    private readonly logger?;
+    private readonly timeoutMs;
+    private readonly codec;
+    constructor(connection: Nats.NatsConnection, subject: string, logger?: FlowcoreLogger | undefined, timeoutMs?: number);
+    distribute(events: FlowcoreEvent[]): Promise<void>;
+}
+declare class NatsDistributionWorker {
+    private readonly connection;
+    private readonly subject;
+    private readonly handler;
+    private readonly logger?;
+    private subscription?;
+    private readonly codec;
+    private readonly queueGroup;
+    constructor(connection: Nats.NatsConnection, subject: string, handler: (events: FlowcoreEvent[]) => Promise<void>, logger?: FlowcoreLogger | undefined);
+    start(): void;
+    stop(): void;
+    private handleMessage;
+}
+
+interface WsEventsMessage {
+    type: "events";
+    deliveryId: string;
+    events: FlowcoreEvent[];
+}
+interface WsAckMessage {
+    type: "ack";
+    deliveryId: string;
+    eventIds: string[];
+}
+interface WsFailMessage {
+    type: "fail";
+    deliveryId: string;
+    eventIds: string[];
+}
+interface WsPingMessage {
+    type: "ping";
+}
+interface WsPongMessage {
+    type: "pong";
+}
+type WsMessage = WsEventsMessage | WsAckMessage | WsFailMessage | WsPingMessage | WsPongMessage;
+declare function serializeMessage(msg: WsMessage): string;
+declare function deserializeMessage(data: string): WsMessage | null;
+interface WsConnectionOptions {
+    logger?: FlowcoreLogger;
+    onMessage: (msg: WsMessage) => void;
+    onClose: () => void;
+}
+declare class WsConnection {
+    private readonly ws;
+    private readonly options;
+    private pingInterval?;
+    private pongTimeout?;
+    private closed;
+    constructor(ws: WebSocket, options: WsConnectionOptions);
+    send(msg: WsMessage): void;
+    close(): void;
+    get isClosed(): boolean;
+    private startPingLoop;
+    private clearPongTimeout;
+    private cleanup;
+}
+interface PendingDelivery {
+    deliveryId: string;
+    eventIds: string[];
+    resolve: () => void;
+    reject: (error: Error) => void;
+    timeout: ReturnType<typeof setTimeout>;
+}
+declare class DeliveryTracker {
+    private pending;
+    add(deliveryId: string, eventIds: string[], timeoutMs: number): Promise<void>;
+    ack(deliveryId: string): void;
+    fail(deliveryId: string): void;
+    rejectAll(error: Error): void;
+    get size(): number;
+    getPendingEventIds(): string[];
+}
+
+declare const noOpLogger: FlowcoreLogger;
+
+export { DeliveryTracker, FlowcoreDataPump, type FlowcoreDataPumpAuth, FlowcoreDataPumpCluster, type FlowcoreDataPumpClusterOptions, type FlowcoreDataPumpCoordinator, type FlowcoreDataPumpDataSource, type FlowcoreDataPumpOptions, type FlowcoreDataPumpProcessor, type FlowcoreDataPumpState, type FlowcoreDataPumpStateManager, FlowcoreDataSource, type FlowcoreDataSourceOptions, type FlowcoreLogger, NatsConnectionManager, NatsDistributionLeader, type NatsDistributionReply, type NatsDistributionRequest, NatsDistributionWorker, type PendingDelivery, PulseEmitter, type PulseEmitterOptions, type PulseLogLevel, type PulseSnapshot, type WsAckMessage, WsConnection, type WsConnectionOptions, type WsEventsMessage, type WsFailMessage, type WsMessage, type WsPingMessage, type WsPongMessage, clusterMetrics, dataPumpPromRegistry, deserializeMessage, metrics, noOpLogger, serializeMessage };