lakesync 0.1.0

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/json-dYtqiL0F.d.ts

@@ -0,0 +1,18 @@
+/**
+ * BigInt-safe JSON replacer.
+ *
+ * Converts BigInt values to strings so they survive `JSON.stringify`,
+ * which otherwise throws on BigInt.
+ */
+declare function bigintReplacer(_key: string, value: unknown): unknown;
+/**
+ * BigInt-aware JSON reviver.
+ *
+ * Restores string-encoded HLC timestamps (fields ending in `Hlc` or `hlc`)
+ * back to BigInt so they match the branded `HLCTimestamp` type.
+ *
+ * Invalid numeric strings are left as-is to prevent runtime crashes.
+ */
+declare function bigintReviver(key: string, value: unknown): unknown;
+
+export { bigintReviver as a, bigintReplacer as b };

package/dist/nessie-client-DrNikVXy.d.ts

@@ -0,0 +1,160 @@
+import { L as LakeSyncError, R as Result } from './result-CojzlFE2.js';
+
+/** Configuration for connecting to a Nessie Iceberg REST catalogue */
+interface CatalogueConfig {
+    /** Nessie Iceberg REST API base URI, e.g. "http://localhost:19120/iceberg" */
+    nessieUri: string;
+    /** Object storage warehouse URI, e.g. "s3://lakesync-warehouse" */
+    warehouseUri: string;
+    /** Nessie branch name. Defaults to "main". */
+    defaultBranch?: string;
+}
+/** Iceberg schema definition following the Iceberg REST spec */
+interface IcebergSchema {
+    type: "struct";
+    "schema-id": number;
+    fields: IcebergField[];
+}
+/** A single field within an Iceberg schema */
+interface IcebergField {
+    id: number;
+    name: string;
+    required: boolean;
+    type: string;
+}
+/** Iceberg partition specification */
+interface PartitionSpec {
+    "spec-id": number;
+    fields: Array<{
+        "source-id": number;
+        "field-id": number;
+        name: string;
+        transform: string;
+    }>;
+}
+/** A data file reference for Iceberg table commits */
+interface DataFile {
+    content: "data";
+    "file-path": string;
+    "file-format": "PARQUET";
+    "record-count": number;
+    "file-size-in-bytes": number;
+    partition?: Record<string, string>;
+}
+/** An Iceberg table snapshot */
+interface Snapshot {
+    "snapshot-id": number;
+    "timestamp-ms": number;
+    summary: Record<string, string>;
+    "manifest-list"?: string;
+}
+/** Full table metadata as returned by the Iceberg REST catalogue */
+interface TableMetadata {
+    "metadata-location"?: string;
+    metadata: {
+        "format-version": number;
+        "table-uuid": string;
+        location: string;
+        "current-schema-id": number;
+        schemas: IcebergSchema[];
+        "current-snapshot-id"?: number;
+        snapshots?: Snapshot[];
+        "partition-specs"?: PartitionSpec[];
+    };
+}
+/** Catalogue operation error */
+declare class CatalogueError extends LakeSyncError {
+    readonly statusCode: number;
+    constructor(message: string, statusCode: number, cause?: Error);
+}
+
+/**
+ * Typed client for the Nessie Iceberg REST Catalogue API v1.
+ *
+ * Wraps standard Iceberg REST endpoints exposed by the Nessie server,
+ * returning `Result<T, CatalogueError>` from every public method.
+ *
+ * On first use, the client fetches `/v1/config` from the server to discover
+ * the catalogue prefix (typically the Nessie branch name, e.g. `"main"`).
+ * All subsequent requests include this prefix in the URL path as required
+ * by the Iceberg REST specification: `/v1/{prefix}/namespaces/...`.
+ */
+declare class NessieCatalogueClient {
+    private readonly baseUri;
+    private readonly warehouseUri;
+    private prefixPromise;
+    constructor(config: CatalogueConfig);
+    /**
+     * Resolve the catalogue prefix by calling the `/v1/config` endpoint.
+     *
+     * The Iceberg REST specification requires a prefix segment in all
+     * API paths (e.g. `/v1/{prefix}/namespaces`). Nessie returns this
+     * value in the `defaults.prefix` field of the config response.
+     *
+     * The result is cached so the config endpoint is only called once
+     * per client instance.
+     *
+     * @returns The resolved prefix string (e.g. `"main"`)
+     */
+    private resolvePrefix;
+    /**
+     * Build the base API path including the resolved prefix.
+     *
+     * @returns URL prefix such as `http://host/iceberg/v1/main` or
+     *          `http://host/iceberg/v1` when no prefix is configured
+     */
+    private apiBase;
+    /**
+     * Create a namespace (idempotent -- ignores 409 Conflict).
+     *
+     * @param namespace - Namespace parts, e.g. `["lakesync"]`
+     * @returns `Ok(void)` on success or if namespace already exists
+     */
+    createNamespace(namespace: string[]): Promise<Result<void, CatalogueError>>;
+    /**
+     * List all namespaces in the catalogue.
+     *
+     * @returns Array of namespace arrays, e.g. `[["lakesync"], ["other"]]`
+     */
+    listNamespaces(): Promise<Result<string[][], CatalogueError>>;
+    /**
+     * Create an Iceberg table within a namespace.
+     *
+     * @param namespace - Namespace parts, e.g. `["lakesync"]`
+     * @param name - Table name
+     * @param schema - Iceberg schema definition
+     * @param partitionSpec - Partition specification
+     */
+    createTable(namespace: string[], name: string, schema: IcebergSchema, partitionSpec: PartitionSpec): Promise<Result<void, CatalogueError>>;
+    /**
+     * Load table metadata from the catalogue.
+     *
+     * @param namespace - Namespace parts, e.g. `["lakesync"]`
+     * @param name - Table name
+     * @returns Full table metadata including schemas, snapshots, and partition specs
+     */
+    loadTable(namespace: string[], name: string): Promise<Result<TableMetadata, CatalogueError>>;
+    /**
+     * Append data files to a table, creating a new snapshot.
+     *
+     * Uses the standard Iceberg REST v1 commit-table endpoint with
+     * `add-snapshot` and `set-snapshot-ref` metadata updates.
+     * First loads the current table metadata to determine the current state,
+     * then commits a new snapshot referencing the provided data files.
+     *
+     * @param namespace - Namespace parts, e.g. `["lakesync"]`
+     * @param table - Table name
+     * @param files - Data files to append
+     */
+    appendFiles(namespace: string[], table: string, files: DataFile[]): Promise<Result<void, CatalogueError>>;
+    /**
+     * Get the current snapshot of a table, or null if no snapshots exist.
+     *
+     * @param namespace - Namespace parts, e.g. `["lakesync"]`
+     * @param table - Table name
+     * @returns The current snapshot, or `null` if the table has no snapshots
+     */
+    currentSnapshot(namespace: string[], table: string): Promise<Result<Snapshot | null, CatalogueError>>;
+}
+
+export { type CatalogueConfig as C, type DataFile as D, type IcebergSchema as I, NessieCatalogueClient as N, type PartitionSpec as P, type Snapshot as S, type TableMetadata as T, CatalogueError as a, type IcebergField as b };

package/dist/parquet.d.ts

@@ -0,0 +1,78 @@
+import { T as TableSchema, R as RowDelta } from './types-V_jVu2sA.js';
+import * as arrow from 'apache-arrow';
+import { R as Result, F as FlushError } from './result-CojzlFE2.js';
+
+/**
+ * Builds an Apache Arrow Schema from a LakeSync TableSchema.
+ *
+ * The resulting schema always includes the following system columns:
+ * - `op` (Utf8) — the delta operation type (INSERT, UPDATE, DELETE)
+ * - `table` (Utf8) — the table name
+ * - `rowId` (Utf8) — the row identifier
+ * - `clientId` (Utf8) — the client identifier
+ * - `hlc` (Int64) — the HLC timestamp as a 64-bit integer
+ * - `deltaId` (Utf8) — the deterministic delta identifier
+ *
+ * User-defined columns from the TableSchema are appended after system columns,
+ * with types mapped as follows:
+ * - `string` → Utf8
+ * - `number` → Float64
+ * - `boolean` → Bool
+ * - `json` → Utf8 (values are JSON-serialised)
+ * - `null` → Utf8, nullable
+ *
+ * @param schema - The LakeSync TableSchema to convert
+ * @returns An Apache Arrow Schema with system and user columns
+ */
+declare function buildArrowSchema(schema: TableSchema): arrow.Schema;
+/**
+ * Converts an array of RowDelta objects into an Apache Arrow Table.
+ *
+ * System columns (op, table, rowId, clientId, hlc, deltaId) are extracted
+ * directly from each delta. User columns are looked up from each delta's
+ * `columns` array; missing columns produce `null` for that row.
+ *
+ * Type conversions:
+ * - `json` columns are serialised via `JSON.stringify`
+ * - `hlc` is passed as a bigint directly to Int64 vectors
+ * - All other types are passed through as-is
+ *
+ * @param deltas - Array of RowDelta objects to convert
+ * @param schema - The LakeSync TableSchema describing user columns
+ * @returns An Apache Arrow Table containing all deltas
+ */
+declare function deltasToArrowTable(deltas: RowDelta[], schema: TableSchema): arrow.Table;
+
+/**
+ * Deserialises Parquet bytes back into an array of RowDelta objects.
+ *
+ * Reads the Parquet data using parquet-wasm, converts to an Apache Arrow Table
+ * via IPC stream, then iterates over rows to reconstruct RowDelta objects.
+ * Int64 `hlc` values are cast back to branded HLCTimestamp bigints.
+ * Columns stored as JSON-serialised Utf8 strings (objects and arrays) are
+ * automatically parsed back to their original values.
+ * Int8 columns marked as booleans in metadata are converted back to true/false.
+ *
+ * @param data - The Parquet file bytes to deserialise
+ * @returns A Result containing the reconstructed RowDelta array, or a FlushError on failure
+ */
+declare function readParquetToDeltas(data: Uint8Array): Promise<Result<RowDelta[], FlushError>>;
+
+/**
+ * Serialises an array of RowDelta objects into Parquet bytes.
+ *
+ * Converts deltas to an Apache Arrow Table via `deltasToArrowTable`,
+ * then encodes the Arrow data as IPC stream bytes and writes them
+ * to Parquet format using Snappy compression via parquet-wasm.
+ *
+ * Boolean columns are stored as Int8 (1/0/null) to work around an
+ * Arrow JS IPC serialisation limitation. The original column types
+ * are preserved in Parquet metadata for the reader to restore.
+ *
+ * @param deltas - The row deltas to serialise
+ * @param schema - The table schema describing user-defined columns
+ * @returns A Result containing the Parquet file as a Uint8Array, or a FlushError on failure
+ */
+declare function writeDeltasToParquet(deltas: RowDelta[], schema: TableSchema): Promise<Result<Uint8Array, FlushError>>;
+
+export { buildArrowSchema, deltasToArrowTable, readParquetToDeltas, writeDeltasToParquet };

package/dist/parquet.js

@@ -0,0 +1,15 @@
+import {
+  buildArrowSchema,
+  deltasToArrowTable,
+  readParquetToDeltas,
+  writeDeltasToParquet
+} from "./chunk-4ARO6KTJ.js";
+import "./chunk-ICNT7I3K.js";
+import "./chunk-7D4SUZUM.js";
+export {
+  buildArrowSchema,
+  deltasToArrowTable,
+  readParquetToDeltas,
+  writeDeltasToParquet
+};
+//# sourceMappingURL=parquet.js.map

package/dist/parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}