lakesync 0.1.0

package/dist/analyst.d.ts

@@ -0,0 +1,268 @@
+import { R as Result, L as LakeSyncError, H as HLCTimestamp } from './result-CojzlFE2.js';
+
+/**
+ * Configuration options for the DuckDB-Wasm client.
+ */
+interface DuckDBClientConfig {
+    /** Whether to enable console logging from DuckDB. Defaults to false. */
+    logger?: boolean;
+    /** Maximum number of threads for DuckDB. Defaults to 1. */
+    threads?: number;
+}
+/**
+ * Wrapper around DuckDB-Wasm that provides a simplified, Result-based API
+ * for executing SQL queries and registering Parquet data sources.
+ *
+ * Works in both Node.js/Bun (using the blocking bindings) and browser
+ * environments (using the async worker-based bindings).
+ *
+ * @example
+ * ```ts
+ * const client = new DuckDBClient({ logger: false });
+ * const initResult = await client.init();
+ * if (!initResult.ok) { console.error(initResult.error); return; }
+ *
+ * const result = await client.query<{ answer: number }>("SELECT 42 AS answer");
+ * if (result.ok) console.log(result.value); // [{ answer: 42 }]
+ *
+ * await client.close();
+ * ```
+ */
+declare class DuckDBClient {
+    private readonly _config;
+    private _db;
+    private _conn;
+    private _closed;
+    constructor(config?: DuckDBClientConfig);
+    /**
+     * Initialise the DuckDB-Wasm instance and open a connection.
+     *
+     * Uses the blocking Node.js bindings when running in Node/Bun,
+     * which avoids the need for Worker threads.
+     *
+     * @returns A Result indicating success or failure with a LakeSyncError
+     */
+    init(): Promise<Result<void, LakeSyncError>>;
+    /**
+     * Execute a SQL query and return the results as an array of objects.
+     *
+     * @param sql - The SQL statement to execute
+     * @param _params - Reserved for future use (parameterised queries)
+     * @returns A Result containing the query results or a LakeSyncError
+     */
+    query<T>(sql: string, _params?: unknown[]): Promise<Result<T[], LakeSyncError>>;
+    /**
+     * Register an in-memory Parquet file as a named table that can be
+     * queried using `SELECT * FROM '<name>'`.
+     *
+     * @param name - The virtual file name (e.g. "deltas.parquet")
+     * @param data - The Parquet file contents as a Uint8Array
+     * @returns A Result indicating success or failure
+     */
+    registerParquetBuffer(name: string, data: Uint8Array): Promise<Result<void, LakeSyncError>>;
+    /**
+     * Register a remote Parquet file by URL so it can be queried using
+     * `SELECT * FROM '<name>'`.
+     *
+     * @param name - The virtual file name (e.g. "remote.parquet")
+     * @param url - The URL pointing to the Parquet file
+     * @returns A Result indicating success or failure
+     */
+    registerParquetUrl(name: string, url: string): Promise<Result<void, LakeSyncError>>;
+    /**
+     * Tear down the DuckDB connection and database instance.
+     *
+     * After calling close(), any subsequent query or registration calls
+     * will return an error Result.
+     */
+    close(): Promise<void>;
+}
+
+/**
+ * Configuration for the TimeTraveller.
+ */
+interface TimeTravelConfig {
+    /** The DuckDB client used for executing time-travel queries. */
+    duckdb: DuckDBClient;
+}
+/**
+ * Provides time-travel query capabilities over delta Parquet files.
+ *
+ * Allows querying the materialised state of data as it existed at a specific
+ * HLC timestamp, or inspecting raw deltas within a time range. Uses DuckDB
+ * SQL with window functions to perform column-level LWW materialisation
+ * entirely in-engine.
+ *
+ * Delta Parquet files contain flattened rows with system columns (`op`, `table`,
+ * `rowId`, `clientId`, `hlc`, `deltaId`) and user-defined columns (e.g. `title`,
+ * `completed`). The materialisation reconstructs per-row state by selecting the
+ * latest value for each column based on HLC ordering, then excluding deleted rows.
+ *
+ * @example
+ * ```ts
+ * const traveller = new TimeTraveller({ duckdb: client });
+ * await traveller.registerDeltas([{ name: "batch-1.parquet", data: bytes }]);
+ *
+ * const result = await traveller.queryAsOf(hlcTimestamp, "SELECT * FROM _state WHERE completed = true");
+ * if (result.ok) console.log(result.value);
+ * ```
+ */
+declare class TimeTraveller {
+    private readonly _config;
+    private readonly _sources;
+    constructor(config: TimeTravelConfig);
+    /**
+     * Register one or more Parquet buffers containing delta data.
+     *
+     * Each buffer is registered with DuckDB and can subsequently be
+     * queried via the time-travel methods.
+     *
+     * @param parquetBuffers - Array of named Parquet file buffers to register
+     * @returns A Result indicating success or a LakeSyncError on failure
+     */
+    registerDeltas(parquetBuffers: Array<{
+        name: string;
+        data: Uint8Array;
+    }>): Promise<Result<void, LakeSyncError>>;
+    /**
+     * Query the materialised state as of the given HLC timestamp.
+     *
+     * Filters all deltas where `hlc <= asOfHlc`, then materialises the latest
+     * state per (table, rowId) using column-level LWW (highest HLC wins per
+     * column). The user's SQL is applied on top of the materialised view,
+     * which is exposed as the CTE `_state`.
+     *
+     * Deleted rows (where the latest operation is DELETE) are excluded from
+     * the materialised view.
+     *
+     * @param asOfHlc - The HLC timestamp representing the point in time to query
+     * @param sql - SQL to apply on the materialised view (use `_state` as the table name)
+     * @returns A Result containing the query results or a LakeSyncError
+     */
+    queryAsOf(asOfHlc: HLCTimestamp, sql: string): Promise<Result<Record<string, unknown>[], LakeSyncError>>;
+    /**
+     * Query raw deltas within a time range.
+     *
+     * Filters deltas where `fromHlc < hlc <= toHlc` and returns them as raw
+     * (unmaterialised) rows. Useful for audit trails and changelog views.
+     *
+     * The user's SQL is applied on top of the filtered deltas, which are
+     * exposed as the CTE `_deltas`.
+     *
+     * @param fromHlc - The exclusive lower bound HLC timestamp
+     * @param toHlc - The inclusive upper bound HLC timestamp
+     * @param sql - SQL to apply on the filtered deltas (use `_deltas` as the table name)
+     * @returns A Result containing the query results or a LakeSyncError
+     */
+    queryBetween(fromHlc: HLCTimestamp, toHlc: HLCTimestamp, sql: string): Promise<Result<Record<string, unknown>[], LakeSyncError>>;
+    /**
+     * Materialise the full state at a point in time, returning all rows.
+     *
+     * Equivalent to `queryAsOf(asOfHlc, "SELECT * FROM _state")` but provided
+     * as a convenience method.
+     *
+     * @param asOfHlc - The HLC timestamp representing the point in time to materialise
+     * @returns A Result containing all materialised rows or a LakeSyncError
+     */
+    materialiseAsOf(asOfHlc: HLCTimestamp): Promise<Result<Record<string, unknown>[], LakeSyncError>>;
+    /**
+     * Build a UNION ALL SQL expression covering all registered Parquet sources.
+     */
+    private _buildUnionSql;
+    /**
+     * Discover user-defined column names from the registered Parquet data.
+     *
+     * Reads the column names from the first registered source and filters
+     * out system columns to identify user-defined columns.
+     */
+    private _discoverUserColumns;
+    /**
+     * Build the materialisation SQL that reconstructs per-row state using
+     * column-level LWW semantics.
+     *
+     * Strategy:
+     * 1. Filter deltas by HLC <= asOfHlc
+     * 2. For each (table, rowId), determine the latest operation (by max HLC)
+     * 3. For each user column in each row, pick the value from the delta with
+     *    the highest HLC (where that column is not null)
+     * 4. Exclude rows where the latest operation is DELETE
+     *
+     * @param userColumns - Names of user-defined columns
+     * @param asOfHlc - The HLC timestamp cutoff as a bigint
+     * @returns SQL string producing the materialised view
+     */
+    private _buildMaterialiseSql;
+}
+
+/**
+ * Configuration for the UnionReader.
+ */
+interface UnionReadConfig {
+    /** The DuckDB client used to query cold (Parquet) data. */
+    duckdb: DuckDBClient;
+    /** The logical table name being queried. */
+    tableName: string;
+}
+/**
+ * Merges "hot" in-memory rows with "cold" Parquet data via DuckDB.
+ *
+ * Cold data is registered as Parquet file buffers in DuckDB. Hot data is
+ * serialised to JSON and loaded via `read_json_auto`. The two sources are
+ * combined with `UNION ALL` and the caller's SQL is applied on top.
+ *
+ * The union is exposed as a CTE named `_union`, so the caller's SQL should
+ * reference `_union` as the table name.
+ *
+ * @example
+ * ```ts
+ * const reader = new UnionReader({ duckdb: client, tableName: "todos" });
+ * await reader.registerColdData([{ name: "batch-1.parquet", data: parquetBytes }]);
+ *
+ * const result = await reader.query(
+ *   "SELECT * FROM _union WHERE completed = true",
+ *   [{ id: "row-3", title: "New task", completed: false }],
+ * );
+ * ```
+ */
+declare class UnionReader {
+    private readonly _config;
+    private readonly _coldSources;
+    private _hotCounter;
+    constructor(config: UnionReadConfig);
+    /**
+     * Register one or more Parquet buffers as cold data sources.
+     *
+     * Each buffer is registered with DuckDB and can subsequently be
+     * queried alongside hot data via {@link query}.
+     *
+     * @param parquetBuffers - Array of named Parquet file buffers to register
+     * @returns A Result indicating success or a LakeSyncError on failure
+     */
+    registerColdData(parquetBuffers: Array<{
+        name: string;
+        data: Uint8Array;
+    }>): Promise<Result<void, LakeSyncError>>;
+    /**
+     * Execute a SQL query that unions hot in-memory rows with cold Parquet data.
+     *
+     * The caller's SQL is wrapped around a UNION ALL of cold and hot sources.
+     * The unioned data is available as `_union` in the SQL statement.
+     *
+     * If `hotRows` is empty or not provided, only cold data is queried.
+     * If no cold sources are registered and `hotRows` is provided, only hot data is queried.
+     *
+     * @param sql - SQL to apply on top of the unioned data (use `_union` as the table name)
+     * @param hotRows - Optional array of in-memory row objects to include in the union
+     * @returns A Result containing the query results or a LakeSyncError
+     */
+    query(sql: string, hotRows?: Record<string, unknown>[]): Promise<Result<Record<string, unknown>[], LakeSyncError>>;
+    /**
+     * Query only cold (Parquet) data without any hot rows.
+     *
+     * @param sql - SQL to execute against cold data (use `_union` as the table name)
+     * @returns A Result containing the query results or a LakeSyncError
+     */
+    queryColdOnly(sql: string): Promise<Result<Record<string, unknown>[], LakeSyncError>>;
+}
+
+export { DuckDBClient, type DuckDBClientConfig, type TimeTravelConfig, TimeTraveller, type UnionReadConfig, UnionReader };