lakesync 0.1.8 → 0.2.0

package/dist/gateway-server.d.ts

@@ -1,15 +1,15 @@
-import { A as AuthClaims } from './auth-CAVutXzx.js';
-export { a as AuthError, v as verifyToken } from './auth-CAVutXzx.js';
+import { A as AuthClaims, b as PollerRegistry } from './create-poller-Cc2MGfhh.js';
+export { a as AuthError, v as verifyToken } from './create-poller-Cc2MGfhh.js';
 import { IncomingMessage, ServerResponse, Server } from 'node:http';
-import { D as DatabaseAdapter } from './db-types-CfLMUBfW.js';
-import { C as ConfigStore, d as SyncGateway, c as HandlerResult } from './request-handler-pUvL7ozF.js';
+import { D as DatabaseAdapter, L as LakeAdapter } from './adapter-types-DwsQGQS4.js';
+import { a as AdapterFactoryRegistry } from './factory-DFfR-030.js';
+import { b as ConfigStore, g as SyncGateway, d as HandlerResult } from './request-handler-B1I5xDOx.js';
 import { R as RowDelta, f as SyncResponse } from './types-BdGBv2ba.js';
-import { L as LakeAdapter } from './types-DSC_EiwR.js';
-import { H as HLCTimestamp } from './result-CojzlFE2.js';
-import './types-Bs-QyOe-.js';
-import './types-BrcD1oJg.js';
-import './base-poller-Bj9kX9dv.js';
+import { R as Result, H as HLCTimestamp } from './result-CojzlFE2.js';
+import './base-poller-Y7ORYgUv.js';
 import './hlc-DiD8QNG3.js';
+import './types-D2C9jTbL.js';
+import './types-Bs-QyOe-.js';
 import './nessie-client-DrNikVXy.js';
 
 /** Result of authentication: either authenticated claims or an error. */
@@ -78,23 +78,82 @@ declare class AdapterBasedLock implements DistributedLock {
     private makeHlc;
 }
 
+/**
+ * Persistence interface for buffering unflushed deltas across restarts.
+ *
+ * Implementations must be synchronous (no async) to avoid race conditions
+ * during the push-then-flush cycle.
+ */
+interface DeltaPersistence {
+    /** Append a batch of deltas to the persistence store. */
+    appendBatch(deltas: RowDelta[]): void;
+    /** Load all persisted deltas. */
+    loadAll(): RowDelta[];
+    /** Clear all persisted deltas (after successful flush). */
+    clear(): void;
+    /** Release resources. */
+    close(): void;
+    /** Persist the cursor state for a connector. */
+    saveCursor(connectorName: string, cursor: string): void;
+    /** Load the persisted cursor state for a connector, or null if none exists. */
+    loadCursor(connectorName: string): string | null;
+}
+/**
+ * In-memory persistence (no durability across restarts).
+ * Used as the default when `persistence` is "memory".
+ */
+declare class MemoryPersistence implements DeltaPersistence {
+    private buffer;
+    private cursors;
+    appendBatch(deltas: RowDelta[]): void;
+    loadAll(): RowDelta[];
+    clear(): void;
+    close(): void;
+    saveCursor(connectorName: string, cursor: string): void;
+    loadCursor(connectorName: string): string | null;
+}
+/**
+ * SQLite-backed persistence using `better-sqlite3`.
+ *
+ * Stores deltas as JSON rows in a single table. On startup, loads all
+ * rows back as RowDeltas. On flush success, truncates the table.
+ */
+declare class SqlitePersistence implements DeltaPersistence {
+    private db;
+    constructor(path: string);
+    appendBatch(deltas: RowDelta[]): void;
+    loadAll(): RowDelta[];
+    clear(): void;
+    saveCursor(connectorName: string, cursor: string): void;
+    loadCursor(connectorName: string): string | null;
+    close(): void;
+}
+
 /**
  * Manages connector registration, adapter creation, and poller lifecycle.
  *
- * Encapsulates the if/else chains for different connector types and
- * handles clean-up on unregistration.
+ * Uses {@link PollerRegistry} and {@link AdapterFactoryRegistry} for
+ * dispatch — no if/else chains for specific connector types.
  */
 declare class ConnectorManager {
     private readonly configStore;
     private readonly gateway;
     private readonly adapters;
     private readonly pollers;
-    constructor(configStore: ConfigStore, gateway: SyncGateway);
+    private readonly pollerRegistry;
+    private readonly adapterRegistry;
+    private readonly persistence;
+    constructor(configStore: ConfigStore, gateway: SyncGateway, options?: {
+        pollerRegistry?: PollerRegistry;
+        adapterRegistry?: AdapterFactoryRegistry;
+        persistence?: DeltaPersistence;
+    });
     /**
      * Register a connector from raw JSON body.
      *
-     * Validates via shared handler, creates adapter/poller as appropriate,
-     * registers with the gateway.
+     * Validates via shared handler, then dispatches to the appropriate
+     * registry: poller registry for API-based connectors, adapter registry
+     * for database-based connectors.
      */
     register(raw: string): Promise<HandlerResult>;
     /**
@@ -183,6 +242,8 @@ declare class SourcePoller {
     private cursorStates;
     /** Diff snapshot per table (keyed by table name). */
     private diffStates;
+    /** Optional callback invoked after each poll with the current cursor state. */
+    onCursorUpdate?: (state: Record<string, unknown>) => void;
     constructor(config: IngestSourceConfig, gateway: SyncGateway);
     /** Start the polling loop. */
     start(): void;
@@ -191,52 +252,205 @@ declare class SourcePoller {
     /** Whether the poller is currently running. */
     get isRunning(): boolean;
     private schedulePoll;
+    /** Export cursor state as a JSON-serialisable object for external persistence. */
+    getCursorState(): Record<string, unknown>;
+    /** Restore cursor state from a previously exported snapshot. */
+    setCursorState(state: Record<string, unknown>): void;
     /** Execute a single poll cycle across all configured tables. */
     poll(): Promise<void>;
     private pollCursor;
     private pollDiff;
 }
 
+/** Supported log levels, ordered by severity. */
+type LogLevel = "debug" | "info" | "warn" | "error";
+/** A single structured log entry. */
+interface LogEntry {
+    level: LogLevel;
+    msg: string;
+    ts: string;
+    [key: string]: unknown;
+}
 /**
- * Persistence interface for buffering unflushed deltas across restarts.
+ * Minimal structured logger that outputs JSON lines to stdout.
  *
- * Implementations must be synchronous (no async) to avoid race conditions
- * during the push-then-flush cycle.
+ * Supports log-level filtering and child loggers with bound context.
+ * No external dependencies — compatible with any log aggregator that
+ * consumes JSON lines.
+ *
+ * @example
+ * ```ts
+ * const logger = new Logger("info");
+ * const reqLogger = logger.child({ requestId: "abc-123" });
+ * reqLogger.info("push received", { deltas: 5 });
+ * // => {"level":"info","msg":"push received","ts":"...","requestId":"abc-123","deltas":5}
+ * ```
  */
-interface DeltaPersistence {
-    /** Append a batch of deltas to the persistence store. */
-    appendBatch(deltas: RowDelta[]): void;
-    /** Load all persisted deltas. */
-    loadAll(): RowDelta[];
-    /** Clear all persisted deltas (after successful flush). */
-    clear(): void;
-    /** Release resources. */
-    close(): void;
+declare class Logger {
+    private readonly minLevelValue;
+    private readonly bindings;
+    /** Output function — defaults to stdout, overridable for testing. */
+    private readonly writeFn;
+    constructor(minLevel?: LogLevel, bindings?: Record<string, unknown>, writeFn?: (line: string) => void);
+    /** Log at debug level. */
+    debug(msg: string, data?: Record<string, unknown>): void;
+    /** Log at info level. */
+    info(msg: string, data?: Record<string, unknown>): void;
+    /** Log at warn level. */
+    warn(msg: string, data?: Record<string, unknown>): void;
+    /** Log at error level. */
+    error(msg: string, data?: Record<string, unknown>): void;
+    /**
+     * Create a child logger with additional bound context.
+     *
+     * The child inherits the parent's level and write function, plus
+     * merges any parent bindings with the new ones.
+     */
+    child(bindings: Record<string, unknown>): Logger;
+    private log;
+    private minLevelName;
 }
+
+/** Label set for a metric observation. */
+type Labels = Record<string, string>;
 /**
- * In-memory persistence (no durability across restarts).
- * Used as the default when `persistence` is "memory".
+ * Monotonically increasing counter.
+ *
+ * @example
+ * ```ts
+ * const pushTotal = new Counter("lakesync_push_total", "Total push requests");
+ * pushTotal.inc({ status: "ok" });
+ * ```
  */
-declare class MemoryPersistence implements DeltaPersistence {
-    private buffer;
-    appendBatch(deltas: RowDelta[]): void;
-    loadAll(): RowDelta[];
-    clear(): void;
-    close(): void;
+declare class Counter {
+    readonly name: string;
+    readonly help: string;
+    private readonly values;
+    constructor(name: string, help: string);
+    /** Increment the counter by `n` (default 1). */
+    inc(labels?: Labels, n?: number): void;
+    /** Return the current value for the given labels. */
+    get(labels?: Labels): number;
+    /** Reset all values. */
+    reset(): void;
+    /** Serialise to Prometheus text exposition format. */
+    expose(): string;
 }
 /**
- * SQLite-backed persistence using `better-sqlite3`.
+ * Gauge that can go up and down.
  *
- * Stores deltas as JSON rows in a single table. On startup, loads all
- * rows back as RowDeltas. On flush success, truncates the table.
+ * @example
+ * ```ts
+ * const bufferBytes = new Gauge("lakesync_buffer_bytes", "Buffer size in bytes");
+ * bufferBytes.set({}, 1024);
+ * ```
  */
-declare class SqlitePersistence implements DeltaPersistence {
-    private db;
-    constructor(path: string);
-    appendBatch(deltas: RowDelta[]): void;
-    loadAll(): RowDelta[];
-    clear(): void;
-    close(): void;
+declare class Gauge {
+    readonly name: string;
+    readonly help: string;
+    private readonly values;
+    constructor(name: string, help: string);
+    /** Set to an absolute value. */
+    set(labels?: Labels, value?: number): void;
+    /** Increment by `n` (default 1). */
+    inc(labels?: Labels, n?: number): void;
+    /** Decrement by `n` (default 1). */
+    dec(labels?: Labels, n?: number): void;
+    /** Return the current value for the given labels. */
+    get(labels?: Labels): number;
+    /** Reset all values. */
+    reset(): void;
+    /** Serialise to Prometheus text exposition format. */
+    expose(): string;
+}
+/**
+ * Histogram with configurable buckets.
+ *
+ * @example
+ * ```ts
+ * const latency = new Histogram(
+ *   "lakesync_push_latency_ms",
+ *   "Push latency in ms",
+ *   [1, 5, 10, 50, 100, 500],
+ * );
+ * latency.observe({}, 42);
+ * ```
+ */
+declare class Histogram {
+    readonly name: string;
+    readonly help: string;
+    readonly buckets: number[];
+    private readonly data;
+    constructor(name: string, help: string, buckets: number[]);
+    /** Record an observation. */
+    observe(labels?: Labels, value?: number): void;
+    /** Return the count of observations for the given labels. */
+    getCount(labels?: Labels): number;
+    /** Return the sum of observations for the given labels. */
+    getSum(labels?: Labels): number;
+    /** Reset all values. */
+    reset(): void;
+    /** Serialise to Prometheus text exposition format. */
+    expose(): string;
+}
+/**
+ * Pre-configured metrics registry for the gateway server.
+ *
+ * Exposes all standard lakesync metrics and a single `expose()` method
+ * that returns the complete Prometheus text exposition payload.
+ */
+declare class MetricsRegistry {
+    readonly pushTotal: Counter;
+    readonly pullTotal: Counter;
+    readonly flushTotal: Counter;
+    readonly flushDuration: Histogram;
+    readonly pushLatency: Histogram;
+    readonly bufferBytes: Gauge;
+    readonly bufferDeltas: Gauge;
+    readonly wsConnections: Gauge;
+    readonly activeRequests: Gauge;
+    /** Return the full Prometheus text exposition payload. */
+    expose(): string;
+    /** Reset all metrics. */
+    reset(): void;
+}
+
+/** Configuration for the per-client rate limiter. */
+interface RateLimiterConfig {
+    /** Maximum requests per window (default: 100). */
+    maxRequests?: number;
+    /** Window size in milliseconds (default: 60_000). */
+    windowMs?: number;
+}
+/**
+ * Fixed-window per-client rate limiter.
+ *
+ * Tracks request counts per client within a sliding window. Stale entries
+ * are periodically cleaned up to prevent unbounded memory growth.
+ */
+declare class RateLimiter {
+    private readonly maxRequests;
+    private readonly windowMs;
+    private readonly clients;
+    private cleanupTimer;
+    constructor(config?: RateLimiterConfig);
+    /**
+     * Attempt to consume one request token for the given client.
+     *
+     * @returns `true` if the request is allowed, `false` if rate-limited.
+     */
+    tryConsume(clientId: string): boolean;
+    /**
+     * Calculate the number of seconds until the current window resets
+     * for a given client. Used for the Retry-After header.
+     */
+    retryAfterSeconds(clientId: string): number;
+    /** Remove all tracked clients and reset state. */
+    reset(): void;
+    /** Stop the periodic cleanup timer. */
+    dispose(): void;
+    /** Remove stale entries whose window has expired. */
+    private cleanup;
 }
 
 /** Matched route information. */
@@ -253,6 +467,96 @@ interface RouteMatch {
  */
 declare function matchRoute(pathname: string, method: string): RouteMatch | null;
 
+/** Consistency mode for shared buffer writes. */
+type ConsistencyMode = "eventual" | "strong";
+/** Configuration for SharedBuffer. */
+interface SharedBufferConfig {
+    /**
+     * Controls how shared adapter write failures are handled.
+     *
+     * - `"eventual"` (default): Shared writes are best-effort. Failures are
+     *   logged but do not fail the push. Local buffer remains authoritative.
+     * - `"strong"`: Shared writes must succeed. Failures are returned as errors,
+     *   allowing the caller to decide whether to fail the push.
+     */
+    consistencyMode?: ConsistencyMode;
+}
+/** Error returned by SharedBuffer in strong consistency mode. */
+interface SharedBufferError {
+    code: "SHARED_WRITE_FAILED";
+    message: string;
+}
+/**
+ * Write-through buffer that pushes to both the in-memory gateway buffer
+ * and a shared database adapter for cross-instance visibility.
+ *
+ * Pull merges in-memory buffer results with adapter query results,
+ * deduplicating by deltaId.
+ */
+declare class SharedBuffer {
+    private readonly sharedAdapter;
+    private readonly consistencyMode;
+    constructor(sharedAdapter: DatabaseAdapter, config?: SharedBufferConfig);
+    /**
+     * Write-through push: write to shared adapter for cross-instance visibility.
+     *
+     * In "eventual" mode (default), failures are logged but do not fail the push.
+     * In "strong" mode, failures are returned as errors.
+     */
+    writeThroughPush(deltas: RowDelta[]): Promise<Result<void, SharedBufferError>>;
+    /**
+     * Merge pull: combine local buffer results with shared adapter results.
+     *
+     * Deduplicates by deltaId to avoid returning the same delta twice.
+     */
+    mergePull(localResult: SyncResponse, sinceHlc: HLCTimestamp): Promise<SyncResponse>;
+}
+
+/** Configuration for WebSocket connection and message rate limits. */
+interface WebSocketLimitsConfig {
+    /** Maximum concurrent WebSocket connections (default: 1000). */
+    maxConnections?: number;
+    /** Maximum messages per second per client (default: 50). */
+    maxMessagesPerSecond?: number;
+}
+/**
+ * Manages WebSocket connections, message handling, and broadcasting.
+ *
+ * Decouples the WebSocket protocol from the HTTP server lifecycle.
+ */
+declare class WebSocketManager {
+    private readonly gateway;
+    private readonly configStore;
+    private readonly gatewayId;
+    private readonly jwtSecret;
+    private readonly wss;
+    private readonly clients;
+    private readonly maxConnections;
+    private readonly maxMessagesPerSecond;
+    private readonly messageRates;
+    private rateResetTimer;
+    constructor(gateway: SyncGateway, configStore: ConfigStore, gatewayId: string, jwtSecret: string | undefined, limits?: WebSocketLimitsConfig);
+    /** The current number of connected clients. */
+    get connectionCount(): number;
+    /** Attach upgrade listener to an HTTP server. */
+    attach(httpServer: Server): void;
+    /**
+     * Broadcast ingested deltas to all connected WebSocket clients except the sender.
+     */
+    broadcastDeltas(deltas: RowDelta[], serverHlc: HLCTimestamp, excludeClientId: string): void;
+    /** Close all connections and shut down the WebSocket server. */
+    close(): void;
+    private handleUpgrade;
+    /**
+     * Check and increment the message rate for a client.
+     * @returns `true` if the message is allowed, `false` if rate-limited.
+     */
+    private checkMessageRate;
+    private handleMessage;
+    private buildSyncRulesContext;
+    private broadcastDeltasAsync;
+}
+
 /** Configuration for the self-hosted gateway server. */
 interface GatewayServerConfig {
     /** Port to listen on (default 3000). */
@@ -283,7 +587,27 @@ interface GatewayServerConfig {
         lock: DistributedLock;
         /** Shared database adapter for cross-instance visibility. */
         sharedAdapter: DatabaseAdapter;
+        /** Shared buffer configuration (consistency mode, etc.). */
+        sharedBufferConfig?: SharedBufferConfig;
     };
+    /** Custom poller registry for connector pollers. Defaults to Jira + Salesforce. */
+    pollerRegistry?: PollerRegistry;
+    /** Custom adapter factory registry. Defaults to Postgres, MySQL, BigQuery. */
+    adapterRegistry?: AdapterFactoryRegistry;
+    /** Drain timeout in milliseconds for graceful shutdown (default 10s). */
+    drainTimeoutMs?: number;
+    /** Request timeout in milliseconds (default 30s). Aborts with 504 on timeout. */
+    requestTimeoutMs?: number;
+    /** Flush timeout in milliseconds for periodic flushes (default 60s). */
+    flushTimeoutMs?: number;
+    /** Per-client rate limiter configuration. When provided, rate limiting is enabled. */
+    rateLimiter?: RateLimiterConfig;
+    /** WebSocket connection and message rate limits. */
+    wsLimits?: WebSocketLimitsConfig;
+    /** Minimum log level for the structured logger (default "info"). */
+    logLevel?: LogLevel;
+    /** Whether to enable the Prometheus metrics endpoint at GET /metrics (default true). */
+    enableMetrics?: boolean;
 }
 /**
  * Self-hosted HTTP gateway server wrapping {@link SyncGateway}.
@@ -310,11 +634,20 @@ declare class GatewayServer {
     private readonly persistence;
     private readonly connectors;
     private readonly sharedBuffer;
+    private readonly rateLimiter;
+    private readonly logger;
+    private readonly metrics;
     private httpServer;
     private wsManager;
     private flushTimer;
     private resolvedPort;
     private pollers;
+    /** Whether the server is draining (rejecting new requests during shutdown). */
+    private draining;
+    /** Number of in-flight requests currently being handled. */
+    private activeRequests;
+    /** Signal handler cleanup functions. */
+    private signalCleanup;
     constructor(config: GatewayServerConfig);
     /**
      * Start the HTTP server and periodic flush timer.
@@ -326,11 +659,17 @@ declare class GatewayServer {
     start(): Promise<void>;
     /** Stop the server, pollers, connectors, and WebSocket connections. */
     stop(): Promise<void>;
+    /** Whether the server is currently draining connections. */
+    get isDraining(): boolean;
     /** The port the server is listening on (available after start). */
     get port(): number;
     /** The underlying SyncGateway instance for direct access. */
     get gatewayInstance(): SyncGateway;
+    /** The Prometheus metrics registry. */
+    get metricsRegistry(): MetricsRegistry;
     private handleRequest;
+    /** Dispatch an authenticated request to the correct route handler. */
+    private dispatchRoute;
     private handlePush;
     private handlePull;
     private handleAction;
@@ -342,59 +681,23 @@ declare class GatewayServer {
     private handleUnregisterConnectorRoute;
     private handleListConnectorsRoute;
     private handleMetricsRoute;
-    private periodicFlush;
-}
-
-/**
- * Write-through buffer that pushes to both the in-memory gateway buffer
- * and a shared database adapter for cross-instance visibility.
- *
- * Pull merges in-memory buffer results with adapter query results,
- * deduplicating by deltaId.
- */
-declare class SharedBuffer {
-    private readonly sharedAdapter;
-    constructor(sharedAdapter: DatabaseAdapter);
+    /** Synchronise buffer gauge metrics with the current buffer state. */
+    private updateBufferGauges;
+    /** Handle GET /ready — checks draining status and adapter health. */
+    private handleReady;
     /**
-     * Write-through push: write to shared adapter for cross-instance visibility.
+     * Check whether the configured adapter is reachable.
      *
-     * Gateway buffer handles fast reads; shared adapter handles
-     * cross-instance visibility and durability.
-     */
-    writeThroughPush(deltas: RowDelta[]): Promise<void>;
-    /**
-     * Merge pull: combine local buffer results with shared adapter results.
-     *
-     * Deduplicates by deltaId to avoid returning the same delta twice.
-     */
-    mergePull(localResult: SyncResponse, sinceHlc: HLCTimestamp): Promise<SyncResponse>;
-}
-
-/**
- * Manages WebSocket connections, message handling, and broadcasting.
- *
- * Decouples the WebSocket protocol from the HTTP server lifecycle.
- */
-declare class WebSocketManager {
-    private readonly gateway;
-    private readonly configStore;
-    private readonly gatewayId;
-    private readonly jwtSecret;
-    private readonly wss;
-    private readonly clients;
-    constructor(gateway: SyncGateway, configStore: ConfigStore, gatewayId: string, jwtSecret: string | undefined);
-    /** Attach upgrade listener to an HTTP server. */
-    attach(httpServer: Server): void;
-    /**
-     * Broadcast ingested deltas to all connected WebSocket clients except the sender.
+     * For a DatabaseAdapter, attempts a lightweight query with a timeout.
+     * For a LakeAdapter, attempts a headObject call (404 still means reachable).
+     * Returns true when no adapter is configured (stateless mode).
      */
-    broadcastDeltas(deltas: RowDelta[], serverHlc: HLCTimestamp, excludeClientId: string): void;
-    /** Close all connections and shut down the WebSocket server. */
-    close(): void;
-    private handleUpgrade;
-    private handleMessage;
-    private buildSyncRulesContext;
-    private broadcastDeltasAsync;
+    private checkAdapterHealth;
+    /** Register SIGTERM/SIGINT handlers for graceful shutdown. */
+    private setupSignalHandlers;
+    /** Graceful shutdown: stop accepting, drain, flush, exit. */
+    private gracefulShutdown;
+    private periodicFlush;
 }
 
-export { AdapterBasedLock, AuthClaims, type AuthResult, ConnectorManager, type CorsConfig, type CursorStrategy, type DeltaPersistence, type DiffStrategy, type DistributedLock, GatewayServer, type GatewayServerConfig, type IngestSourceConfig, type IngestTableConfig, MemoryPersistence, type QueryFn, type RouteMatch, SharedBuffer, SourcePoller, SqlitePersistence, WebSocketManager, authenticateRequest, corsHeaders, extractBearerToken, handlePreflight, matchRoute };
+export { AdapterBasedLock, AuthClaims, type AuthResult, ConnectorManager, type ConsistencyMode, type CorsConfig, Counter, type CursorStrategy, type DeltaPersistence, type DiffStrategy, type DistributedLock, GatewayServer, type GatewayServerConfig, Gauge, Histogram, type IngestSourceConfig, type IngestTableConfig, type Labels, type LogEntry, type LogLevel, Logger, MemoryPersistence, MetricsRegistry, type QueryFn, RateLimiter, type RateLimiterConfig, type RouteMatch, SharedBuffer, type SharedBufferConfig, type SharedBufferError, SourcePoller, SqlitePersistence, type WebSocketLimitsConfig, WebSocketManager, authenticateRequest, corsHeaders, extractBearerToken, handlePreflight, matchRoute };