lakesync 0.1.0

package/README.md

@@ -0,0 +1,74 @@
+# lakesync
+
+Unified npm package for LakeSync -- a single install that provides access to all LakeSync sub-packages via subpath exports.
+
+## Install
+
+```bash
+npm install lakesync
+```
+
+## Usage
+
+The default export re-exports `@lakesync/core`. Each sub-package is available via a subpath:
+
+```ts
+// Core types and utilities (default export)
+import { HLC, extractDelta, Ok, Err } from "lakesync";
+
+// Client SDK
+import { LocalDB, SyncCoordinator, HttpTransport } from "lakesync/client";
+
+// Gateway
+import { SyncGateway, DeltaBuffer } from "lakesync/gateway";
+
+// Storage adapter
+import { MinIOAdapter } from "lakesync/adapter";
+
+// Protobuf codec
+import { encodeSyncPush, decodeSyncPush } from "lakesync/proto";
+
+// Parquet read/write
+import { writeDeltasToParquet, readParquetToDeltas } from "lakesync/parquet";
+
+// Iceberg catalogue
+import { NessieCatalogueClient } from "lakesync/catalogue";
+
+// Compaction
+import { Compactor, MaintenanceRunner } from "lakesync/compactor";
+
+// Analytics
+import { DuckDBClient, UnionReader, TimeTraveller } from "lakesync/analyst";
+```
+
+## Subpath exports
+
+| Subpath | Maps to | Description |
+|---------|---------|-------------|
+| `lakesync` | `@lakesync/core` | HLC, delta types, conflict resolution, Result type |
+| `lakesync/client` | `@lakesync/client` | SyncCoordinator, LocalDB, transports, queues |
+| `lakesync/gateway` | `@lakesync/gateway` | Sync gateway with delta buffer and flush |
+| `lakesync/adapter` | `@lakesync/adapter` | Storage adapter interface + S3/MinIO implementation |
+| `lakesync/proto` | `@lakesync/proto` | Protobuf codec for the wire protocol |
+| `lakesync/parquet` | `@lakesync/parquet` | Parquet read/write via parquet-wasm |
+| `lakesync/catalogue` | `@lakesync/catalogue` | Iceberg REST catalogue client |
+| `lakesync/compactor` | `@lakesync/compactor` | Parquet compaction and maintenance |
+| `lakesync/analyst` | `@lakesync/analyst` | DuckDB-WASM analytics and time-travel queries |
+
+## Peer dependencies
+
+Heavy runtime dependencies are optional peer dependencies -- install only what you need:
+
+| Peer dependency | Required for |
+|-----------------|-------------|
+| `sql.js` | `lakesync/client` (LocalDB) |
+| `idb` | `lakesync/client` (IDBQueue, IDB persistence) |
+| `@aws-sdk/client-s3` | `lakesync/adapter` (MinIOAdapter) |
+| `@bufbuild/protobuf` | `lakesync/proto` |
+| `parquet-wasm`, `apache-arrow` | `lakesync/parquet` |
+| `@duckdb/duckdb-wasm` | `lakesync/analyst` |
+
+## When to use this vs `@lakesync/*`
+
+- **Use `lakesync`** for quick prototyping or when you want a single dependency
+- **Use individual `@lakesync/*` packages** for production to minimise bundle size and control peer dependencies precisely

package/dist/adapter.d.ts

@@ -0,0 +1,369 @@
+import { BigQuery } from '@google-cloud/bigquery';
+import { R as Result, A as AdapterError, H as HLCTimestamp } from './result-CojzlFE2.js';
+import { R as RowDelta, T as TableSchema, C as ColumnDelta } from './types-V_jVu2sA.js';
+import { D as DatabaseAdapter, a as DatabaseAdapterConfig } from './db-types-BR6Kt4uf.js';
+export { i as isDatabaseAdapter, l as lakeSyncTypeToBigQuery } from './db-types-BR6Kt4uf.js';
+import { C as ConnectorConfig } from './types-DAQL_vU_.js';
+import { L as LakeAdapter, A as AdapterConfig, O as ObjectInfo } from './types-DSC_EiwR.js';
+import mysql from 'mysql2/promise';
+import { Pool } from 'pg';
+
+/**
+ * Configuration for the BigQuery adapter.
+ * Unlike SQL adapters, BigQuery is HTTP-based — no connection string needed.
+ */
+interface BigQueryAdapterConfig {
+    /** GCP project ID. */
+    projectId: string;
+    /** BigQuery dataset name. */
+    dataset: string;
+    /** Path to a service account JSON key file. Falls back to ADC if omitted. */
+    keyFilename?: string;
+    /** Dataset location (default: "US"). */
+    location?: string;
+}
+/**
+ * BigQuery database adapter for LakeSync.
+ *
+ * Stores deltas in a `lakesync_deltas` table using standard SQL DML.
+ * Idempotent inserts via MERGE statement. All public methods return
+ * `Result` and never throw.
+ *
+ * **Note:** BigQuery DML is limited to 1,500 statements per table per day
+ * on standard (non-partitioned) tables. Query latency is seconds, not
+ * milliseconds — this adapter is designed for the analytics tier.
+ */
+declare class BigQueryAdapter implements DatabaseAdapter {
+    /** @internal */
+    readonly client: BigQuery;
+    /** @internal */
+    readonly dataset: string;
+    /** @internal */
+    readonly location: string;
+    constructor(config: BigQueryAdapterConfig);
+    /**
+     * Insert deltas into the database in a single batch.
+     * Idempotent via MERGE — existing deltaIds are silently skipped.
+     */
+    insertDeltas(deltas: RowDelta[]): Promise<Result<void, AdapterError>>;
+    /**
+     * Query deltas with HLC greater than the given timestamp, optionally filtered by table.
+     */
+    queryDeltasSince(hlc: HLCTimestamp, tables?: string[]): Promise<Result<RowDelta[], AdapterError>>;
+    /**
+     * Get the latest merged state for a specific row using column-level LWW.
+     * Returns null if no deltas exist for this row.
+     */
+    getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
+    /**
+     * Ensure the BigQuery dataset and lakesync_deltas table exist.
+     * The `schema` parameter is accepted for interface compliance but the
+     * internal table structure is fixed (deltas store column data as JSON).
+     */
+    ensureSchema(_schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /**
+     * No-op — BigQuery client is HTTP-based with no persistent connections.
+     */
+    close(): Promise<void>;
+}
+
+/** A routing rule that maps specific tables to a database adapter. */
+interface CompositeRoute {
+    /** Tables handled by this adapter */
+    tables: string[];
+    /** The adapter for these tables */
+    adapter: DatabaseAdapter;
+}
+/** Configuration for CompositeAdapter routing. */
+interface CompositeAdapterConfig {
+    /** Table-to-adapter routing rules */
+    routes: CompositeRoute[];
+    /** Fallback adapter for tables not matching any route */
+    defaultAdapter: DatabaseAdapter;
+}
+/**
+ * Routes database operations to different adapters based on table name.
+ * Implements DatabaseAdapter so it can be used as a drop-in replacement.
+ */
+declare class CompositeAdapter implements DatabaseAdapter {
+    private readonly routeMap;
+    private readonly adapters;
+    private readonly defaultAdapter;
+    constructor(config: CompositeAdapterConfig);
+    /** Insert deltas, routing each group to the correct adapter by table. */
+    insertDeltas(deltas: RowDelta[]): Promise<Result<void, AdapterError>>;
+    /** Query deltas since a given HLC, fanning out to relevant adapters and merging results. */
+    queryDeltasSince(hlc: HLCTimestamp, tables?: string[]): Promise<Result<RowDelta[], AdapterError>>;
+    /** Get the latest state for a row, routing to the correct adapter. */
+    getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
+    /** Ensure schema exists, routing to the correct adapter for the table. */
+    ensureSchema(schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /** Close all unique adapters (routes + default, deduplicated). */
+    close(): Promise<void>;
+}
+
+/**
+ * Instantiate a {@link DatabaseAdapter} from a {@link ConnectorConfig}.
+ *
+ * Switches on `config.type` and creates the matching adapter using
+ * the type-specific connection configuration. Returns an {@link AdapterError}
+ * if the type-specific config is missing or the adapter constructor throws.
+ *
+ * @param config - Validated connector configuration.
+ * @returns The instantiated adapter or an error.
+ */
+declare function createDatabaseAdapter(config: ConnectorConfig): Result<DatabaseAdapter, AdapterError>;
+
+/** Configuration for the FanOutAdapter. */
+interface FanOutAdapterConfig {
+    /** The primary adapter that handles all reads and authoritative writes. */
+    primary: DatabaseAdapter;
+    /** Secondary adapters that receive replicated writes on a best-effort basis. */
+    secondaries: DatabaseAdapter[];
+}
+/**
+ * Writes to a primary adapter synchronously and replicates to secondary
+ * adapters asynchronously. Reads always go to the primary.
+ *
+ * Secondary failures are silently caught and never affect the return value.
+ * Use case: write to Postgres (fast, operational), replicate to BigQuery (analytics).
+ */
+declare class FanOutAdapter implements DatabaseAdapter {
+    private readonly primary;
+    private readonly secondaries;
+    constructor(config: FanOutAdapterConfig);
+    /** Insert deltas into the primary, then replicate to secondaries (fire-and-forget). */
+    insertDeltas(deltas: RowDelta[]): Promise<Result<void, AdapterError>>;
+    /** Query deltas from the primary adapter only. */
+    queryDeltasSince(hlc: HLCTimestamp, tables?: string[]): Promise<Result<RowDelta[], AdapterError>>;
+    /** Get the latest state from the primary adapter only. */
+    getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
+    /** Ensure schema on the primary first, then best-effort on secondaries. */
+    ensureSchema(schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /** Close primary and all secondary adapters. */
+    close(): Promise<void>;
+}
+
+/** Configuration for age-based tiered storage. */
+interface LifecycleAdapterConfig {
+    /** Hot tier — recent data, fast queries. */
+    hot: {
+        /** The adapter storing recent deltas. */
+        adapter: DatabaseAdapter;
+        /** Maximum age in milliseconds before data is considered cold. */
+        maxAgeMs: number;
+    };
+    /** Cold tier — older data, cheap storage. */
+    cold: {
+        /** The adapter storing archived deltas. */
+        adapter: DatabaseAdapter;
+    };
+}
+/**
+ * Routes database operations across hot and cold tiers based on delta age.
+ *
+ * Writes always go to the hot adapter. Reads fan out to both tiers when
+ * the requested HLC is older than the configured `maxAgeMs` threshold.
+ *
+ * Use {@link migrateToTier} as a background job to copy aged-out deltas
+ * from hot to cold.
+ */
+declare class LifecycleAdapter implements DatabaseAdapter {
+    private readonly hot;
+    private readonly cold;
+    private readonly maxAgeMs;
+    constructor(config: LifecycleAdapterConfig);
+    /** Insert deltas into the hot adapter — new data is always hot. */
+    insertDeltas(deltas: RowDelta[]): Promise<Result<void, AdapterError>>;
+    /**
+     * Query deltas since the given HLC.
+     *
+     * If `sinceHlc` is older than `now - maxAgeMs`, queries both hot and cold
+     * adapters and merges the results sorted by HLC. Otherwise queries hot only.
+     */
+    queryDeltasSince(hlc: HLCTimestamp, tables?: string[]): Promise<Result<RowDelta[], AdapterError>>;
+    /** Get latest state — try hot first, fall back to cold if hot returns null. */
+    getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
+    /** Ensure schema exists on both hot and cold adapters. */
+    ensureSchema(schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /** Close both hot and cold adapters. */
+    close(): Promise<void>;
+}
+/**
+ * Migrate aged-out deltas from the hot adapter to the cold adapter.
+ *
+ * Queries the hot adapter for all deltas since HLC 0, filters those with
+ * wall time older than `Date.now() - maxAgeMs`, and inserts them into the
+ * cold adapter. Insertion is idempotent via deltaId uniqueness.
+ *
+ * Does NOT delete from hot — that is a separate cleanup concern.
+ *
+ * @param hot - The hot-tier adapter to read old deltas from.
+ * @param cold - The cold-tier adapter to write old deltas to.
+ * @param maxAgeMs - Age threshold in milliseconds.
+ * @returns The count of migrated deltas, or an AdapterError.
+ */
+declare function migrateToTier(hot: DatabaseAdapter, cold: DatabaseAdapter, maxAgeMs: number): Promise<Result<{
+    migrated: number;
+}, AdapterError>>;
+
+/** Options for migrating deltas between database adapters. */
+interface MigrateOptions {
+    /** Source adapter to read from */
+    from: DatabaseAdapter;
+    /** Target adapter to write to */
+    to: DatabaseAdapter;
+    /** Optional: only migrate specific tables */
+    tables?: string[];
+    /** Batch size for writing (default: 1000) */
+    batchSize?: number;
+    /** Progress callback invoked after each batch write */
+    onProgress?: (info: MigrateProgress) => void;
+}
+/** Progress information reported during migration. */
+interface MigrateProgress {
+    /** Current batch number (1-based) */
+    batch: number;
+    /** Total deltas migrated so far */
+    totalSoFar: number;
+}
+/** Result of a successful migration. */
+interface MigrateResult {
+    /** Total number of deltas migrated */
+    totalDeltas: number;
+    /** Number of batches processed */
+    batches: number;
+}
+/**
+ * Migrate deltas from one database adapter to another.
+ * Reads all matching deltas from the source, then writes them in batches to the target.
+ * Idempotent via deltaId uniqueness in the target adapter.
+ */
+declare function migrateAdapter(opts: MigrateOptions): Promise<Result<MigrateResult, AdapterError>>;
+
+/**
+ * MinIO/S3-compatible lake adapter.
+ *
+ * Wraps the AWS S3 SDK to provide a Result-based interface for
+ * interacting with MinIO or any S3-compatible object store.
+ * All public methods return `Result` and never throw.
+ */
+declare class MinIOAdapter implements LakeAdapter {
+    private readonly client;
+    private readonly bucket;
+    constructor(config: AdapterConfig);
+    /** Store an object in the lake */
+    putObject(path: string, data: Uint8Array, contentType?: string): Promise<Result<void, AdapterError>>;
+    /** Retrieve an object from the lake */
+    getObject(path: string): Promise<Result<Uint8Array, AdapterError>>;
+    /** Get object metadata without retrieving the body */
+    headObject(path: string): Promise<Result<{
+        size: number;
+        lastModified: Date;
+    }, AdapterError>>;
+    /** List objects matching a given prefix */
+    listObjects(prefix: string): Promise<Result<ObjectInfo[], AdapterError>>;
+    /** Delete a single object from the lake */
+    deleteObject(path: string): Promise<Result<void, AdapterError>>;
+    /** Delete multiple objects from the lake in a single batch operation */
+    deleteObjects(paths: string[]): Promise<Result<void, AdapterError>>;
+}
+
+/**
+ * MySQL database adapter for LakeSync.
+ *
+ * Stores deltas in a `lakesync_deltas` table using INSERT IGNORE for
+ * idempotent writes. All public methods return `Result` and never throw.
+ * Uses mysql2/promise connection pool for async operations.
+ */
+declare class MySQLAdapter implements DatabaseAdapter {
+    /** @internal */
+    readonly pool: mysql.Pool;
+    constructor(config: DatabaseAdapterConfig);
+    /**
+     * Insert deltas into the database in a single batch.
+     * Uses INSERT IGNORE for idempotent writes — duplicate deltaIds are silently skipped.
+     */
+    insertDeltas(deltas: RowDelta[]): Promise<Result<void, AdapterError>>;
+    /**
+     * Query deltas with HLC greater than the given timestamp.
+     * Optionally filtered by table name(s).
+     */
+    queryDeltasSince(hlc: HLCTimestamp, tables?: string[]): Promise<Result<RowDelta[], AdapterError>>;
+    /**
+     * Get the latest merged state for a specific row using column-level LWW.
+     * Returns null if no deltas exist or if the row is tombstoned by DELETE.
+     */
+    getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
+    /**
+     * Ensure the database schema exists. Creates the lakesync_deltas table
+     * and a user table matching the given TableSchema definition.
+     */
+    ensureSchema(schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /** Close the database connection pool and release resources. */
+    close(): Promise<void>;
+}
+
+/**
+ * PostgreSQL database adapter for LakeSync.
+ *
+ * Stores deltas in a `lakesync_deltas` table using pg Pool.
+ * All public methods return `Result` and never throw.
+ */
+declare class PostgresAdapter implements DatabaseAdapter {
+    /** @internal */
+    readonly pool: Pool;
+    constructor(config: DatabaseAdapterConfig);
+    /**
+     * Insert deltas into the database in a single batch.
+     * Idempotent via `ON CONFLICT (delta_id) DO NOTHING`.
+     */
+    insertDeltas(deltas: RowDelta[]): Promise<Result<void, AdapterError>>;
+    /**
+     * Query deltas with HLC greater than the given timestamp, optionally filtered by table.
+     */
+    queryDeltasSince(hlc: HLCTimestamp, tables?: string[]): Promise<Result<RowDelta[], AdapterError>>;
+    /**
+     * Get the latest merged state for a specific row using column-level LWW.
+     * Returns null if no deltas exist for this row.
+     */
+    getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
+    /**
+     * Ensure the lakesync_deltas table and indices exist.
+     * The `schema` parameter is accepted for interface compliance but the
+     * internal table structure is fixed (deltas store column data as JSONB).
+     */
+    ensureSchema(_schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /** Close the database connection pool and release resources. */
+    close(): Promise<void>;
+}
+
+/** Generic query function — abstracts any SQL database connection. */
+type QueryFn = (sql: string, params?: unknown[]) => Promise<Record<string, unknown>[]>;
+/**
+ * Create a raw SQL query function from a {@link ConnectorConfig}.
+ *
+ * Uses dynamic imports so the database drivers (pg, mysql2) are only
+ * loaded when actually needed. Returns `null` for connector types that
+ * do not support the standard SQL polling model (e.g. BigQuery).
+ *
+ * @param config - Validated connector configuration.
+ * @returns A query function or `null` if the connector type is unsupported.
+ */
+declare function createQueryFn(config: ConnectorConfig): Promise<QueryFn | null>;
+
+/** Normalise a caught value into an Error or undefined. */
+declare function toCause(error: unknown): Error | undefined;
+/** Execute an async operation and wrap errors into an AdapterError Result. */
+declare function wrapAsync<T>(operation: () => Promise<T>, errorMessage: string): Promise<Result<T, AdapterError>>;
+/**
+ * Merge delta rows into final state using column-level LWW.
+ * Shared by Postgres, MySQL, and BigQuery getLatestState implementations.
+ * Rows must be sorted by HLC ascending.
+ */
+declare function mergeLatestState(rows: Array<{
+    columns: string | ColumnDelta[];
+    op: string;
+}>): Record<string, unknown> | null;
+
+export { AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, ObjectInfo, PostgresAdapter, type QueryFn, createDatabaseAdapter, createQueryFn, mergeLatestState, migrateAdapter, migrateToTier, toCause, wrapAsync };

package/dist/adapter.js

@@ -0,0 +1,39 @@
+import {
+  BigQueryAdapter,
+  CompositeAdapter,
+  FanOutAdapter,
+  LifecycleAdapter,
+  MinIOAdapter,
+  MySQLAdapter,
+  PostgresAdapter,
+  createDatabaseAdapter,
+  createQueryFn,
+  isDatabaseAdapter,
+  lakeSyncTypeToBigQuery,
+  mergeLatestState,
+  migrateAdapter,
+  migrateToTier,
+  toCause,
+  wrapAsync
+} from "./chunk-X3RO5SYJ.js";
+import "./chunk-ICNT7I3K.js";
+import "./chunk-7D4SUZUM.js";
+export {
+  BigQueryAdapter,
+  CompositeAdapter,
+  FanOutAdapter,
+  LifecycleAdapter,
+  MinIOAdapter,
+  MySQLAdapter,
+  PostgresAdapter,
+  createDatabaseAdapter,
+  createQueryFn,
+  isDatabaseAdapter,
+  lakeSyncTypeToBigQuery,
+  mergeLatestState,
+  migrateAdapter,
+  migrateToTier,
+  toCause,
+  wrapAsync
+};
+//# sourceMappingURL=adapter.js.map

package/dist/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}