@sixb/ducklake 0.1.0 → 0.1.2

package/README.md

@@ -94,15 +94,22 @@ const report = await lakeStorage.runMaintenance({
 Reads, writes, SQL previews, and SQL transforms use the same DuckDB runtime and
 the same DuckLake PostgreSQL metadata pool. Work is serialized inside one
 `DuckLakeStorage` instance, so a queued DuckDB operation makes later operations
-wait. Only actual DuckDB work holds the runtime: batch appends, commits, SQL
-transforms, and metadata reads. A large streaming read can also make a later
-write wait until the stream is consumed or closed.
+wait. Only actual DuckDB work holds the runtime: bounded read pages, batch
+appends, commits, SQL transforms, and metadata reads.
+
+Large `readRows(...)` calls materialize bounded pages, using a physical-row keyset when available,
+and release the DuckDB queue between pages. This also applies when an explicit limit exceeds one
+page. A JavaScript pipeline can therefore stream one DuckLake dataset directly into another on the
+same provider without buffering the complete input or deadlocking at the destination appender's
+first flush. The read remains pinned to one immutable snapshot and keeps its attachment lease until
+the iterator completes or closes.
 
 A write does not hold the runtime while its source iterable is producing rows.
 `writeRows(...)` validates and stages rows in bounded in-memory batches, then
 takes a queue slot only to flush each batch into the staging table. Slow external
 reads (pagination, APIs, SFTP, retries) run outside the queue, so other reads and
-write batches can interleave between a write's batches.
+write batches can interleave between a write's batches. Large in-memory writes
+also yield to the event loop between batches.
 
 ```txt
 same provider instance:
@@ -146,6 +153,43 @@ use the same schema that DuckLake materializes.
 
 DuckLake stores Sixb `decimal` columns as `DECIMAL(38, 9)` and returns them as exact strings, including from SQL previews. Writes that need more than 29 integer digits or 9 fractional digits are rejected before DuckDB can round or overflow them.
 
+## Keyed Merges
+
+Datasets with a primary key can apply ordered complete-row changes directly through the lake
+provider. Key columns must be non-null strings; composite key order is significant.
+
+```ts
+import { change, col, defineDataset } from "@sixb/core"
+
+const invoices = defineDataset("erp.invoices", {
+  schema: [
+    col("id", "string"),
+    col("status", "string"),
+    col("customerId", "string"),
+  ],
+  primaryKey: "id",
+})
+
+await lakeStorage.createDataset(invoices)
+
+const merge = await lakeStorage.beginMerge({ dataset: invoices })
+await merge.writeChanges([
+  change.upsert({ id: "inv_1", status: "paid", customerId: "cust_1" }),
+  change.delete({ id: "inv_2" }),
+])
+const result = await merge.commit()
+```
+
+Changes remain ordered across `writeChanges(...)` calls, and the final change for a repeated key
+wins. Upserts are complete rows rather than patches. Commit checks the dataset version captured by
+`beginMerge`, applies effective changes atomically, and creates a version only when visible rows
+change. An initial no-op returns `{ outcome: "unchanged", version: null }`; a later no-op returns the
+existing latest version.
+
+Snapshot, append, and SQL-transform writes to keyed datasets also enforce unique keys so a later
+merge never starts from an ambiguous baseline. Application authors normally use
+`defineSync(..., { mode: "merge" })`; the sync worker stages those changes through this contract.
+
 ## DuckLake SQL Transforms
 
 DuckLake SQL transforms let the provider run dataset-to-dataset transforms in

package/dist/ducklake-storage.d.ts

@@ -1,5 +1,5 @@
 import type { DatasetDefinition, DatasetRow, LakeStorageWithSql } from "@sixb/core";
-import type { BeginDatasetWriteInput, DatasetCatalogState, DatasetVersion, LakeWriteSession, ReadDatasetRowsInput } from "@sixb/core/lake-storage";
+import type { BeginDatasetMergeInput, BeginDatasetWriteInput, DatasetCatalogState, DatasetVersion, LakeMergeSession, LakeWriteSession, ReadDatasetRowsInput } from "@sixb/core/lake-storage";
 import { DuckLakeSqlExecutor } from "./internal/ducklake-sql-executor";
 import type { DuckLakeMaintenanceOptions, DuckLakeMaintenanceReport, DuckLakeStorageOptions } from "./types";
 /**
@@ -29,6 +29,7 @@ export declare class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
     listDatasetCatalogState(datasetIds: readonly string[]): Promise<readonly DatasetCatalogState[]>;
     listVersions(datasetId: string, limit?: number): Promise<readonly DatasetVersion[]>;
     beginWrite(input: BeginDatasetWriteInput): Promise<LakeWriteSession>;
+    beginMerge(input: BeginDatasetMergeInput): Promise<LakeMergeSession>;
     getLatestVersion(datasetId: string): Promise<DatasetVersion | null>;
     getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null>;
     readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>;

package/dist/ducklake-storage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ducklake-storage.d.ts","sourceRoot":"","sources":["../src/ducklake-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAMhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAA;AAEtE,OAAO,KAAK,EACV,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAEhB;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,kBAAkB,CAAC,QAAQ,CAAC;IAClE,QAAQ,CAAC,QAAQ;;;MAA8C;IAC/D,QAAQ,CAAC,GAAG,EAAE,mBAAmB,CAAA;IAEjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,OAAO,EAAE,sBAAsB;IAmCrC,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIxE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIrD,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;IAIpC,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAInF,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAInE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAItF,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;IAI1D,cAAc,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAI9F;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
+{"version":3,"file":"ducklake-storage.d.ts","sourceRoot":"","sources":["../src/ducklake-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAMhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAA;AAEtE,OAAO,KAAK,EACV,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAEhB;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,kBAAkB,CAAC,QAAQ,CAAC;IAClE,QAAQ,CAAC,QAAQ;;;MAA8C;IAC/D,QAAQ,CAAC,GAAG,EAAE,mBAAmB,CAAA;IAEjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,OAAO,EAAE,sBAAsB;IAmCrC,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIxE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIrD,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;IAIpC,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAInF,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAInE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAItF,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;IAI1D,cAAc,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAI9F;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}