@sixb/ducklake 0.1.5 → 0.1.7

package/README.md

@@ -91,18 +91,24 @@ const report = await lakeStorage.runMaintenance({
 
 ### Read and Write Concurrency
 
-Reads, writes, SQL previews, and SQL transforms use the same DuckDB runtime and
-the same DuckLake PostgreSQL metadata pool. Work is serialized inside one
-`DuckLakeStorage` instance, so a queued DuckDB operation makes later operations
-wait. Only actual DuckDB work holds the runtime: bounded read pages, batch
-appends, commits, SQL transforms, and metadata reads.
-
-Large `readRows(...)` calls materialize bounded pages, using a physical-row keyset when available,
-and release the DuckDB queue between pages. This also applies when an explicit limit exceeds one
-page. A JavaScript pipeline can therefore stream one DuckLake dataset directly into another on the
-same provider without buffering the complete input or deadlocking at the destination appender's
-first flush. The read remains pinned to one immutable snapshot and keeps its attachment lease until
-the iterator completes or closes.
+Large reads stream through dedicated connections in the same DuckDB engine, sharing its memory
+budget and PostgreSQL metadata pool. Metadata and writes remain available between yielded rows.
+Each read stays pinned to one immutable snapshot. Maintenance waits until no reads remain; it
+blocks new reads only while running, so continuous traffic can postpone it.
+
+`maxStreamingReads` limits native streaming readers per provider (default: `4`); extra reads use
+bounded pages on the main connection. It does not limit pipeline inputs. Raise it when memory and
+catalog capacity allow. Finish the iterator, use `for await` with `break`, or pass `signal` to cancel
+it. Projection and pipeline workers propagate their cancellation automatically.
+
+Small previews, SQLite/custom catalogs, and PostgreSQL reads without sufficient pool headroom
+use bounded pages. PostgreSQL streaming admits at most `maxConnections - 4` cursors to leave
+headroom for metadata and writes. Parallel scans can exceed the pool's cache limit, so this does
+not guarantee a server-wide connection count.
+
+Every engine uses its own spill subdirectory under `duckdb.config.temp_directory` (default:
+`.tmp` for an in-memory engine). Closing it removes only that subdirectory. Sharing spill filenames
+between engines can corrupt results. An empty `temp_directory` still disables spilling.
 
 A write does not hold the runtime while its source iterable is producing rows.
 `writeRows(...)` validates and stages rows in bounded in-memory batches, then
@@ -111,18 +117,6 @@ reads (pagination, APIs, SFTP, retries) run outside the queue, so other reads an
 write batches can interleave between a write's batches. Large in-memory writes
 also yield to the event loop between batches.
 
-```txt
-same provider instance:
-  read -> write -> read
-
-the operations run in order on one DuckDB runtime
-```
-
-This is the simplest and most predictable default, especially for small
-PostgreSQL plans. A future provider option could add a bounded read-runtime pool
-for deployments that have enough PostgreSQL capacity and need more concurrent
-read throughput.
-
 Datasets must declare a schema before they can be stored in DuckLake:
 
 ```ts
@@ -196,12 +190,16 @@ Snapshot syncs use `commit({ createInitialVersion: true })` to make a first empt
 addressable. Later no-op commits reuse the current version, including after a concurrent write.
 Use `commit({ retryOnConflict: true })` to rebase retained changes, with at most 3 attempts.
 Explicit `expectedLatestVersionId` guards are never relaxed.
-See [source ordering](../../docs/data/datasets.md#source-ordering).
+See [source ordering](../../docs/datasets/source-ordering.md).
 
 Local catalogs serialize commits within one process. PostgreSQL catalogs support concurrent
 writer processes; known snapshot-ID conflicts are retried from the retained staging table.
 Other provider/connection failures propagate without an automatic retry.
 
+SQLite reads commit metadata before the write transaction, then verifies the native snapshot
+has not changed. Preparation retries up to three times on a race; data writes are not replayed.
+The catalog's existing journal mode is preserved.
+
 Snapshot, append, and SQL-transform writes to keyed datasets also enforce unique keys so a later
 merge never starts from an ambiguous baseline. Application authors normally use
 `defineSync(..., { mode: "merge" })`; the sync worker stages those changes through this contract.
@@ -467,7 +465,7 @@ DuckLake metadata catalog.
 
 | Option | Use it for |
 | --- | --- |
-| `maxConnections` | Hard cap for DuckDB PostgreSQL extension connections. |
+| `maxConnections` | Pool cache limit; parallel scans can temporarily exceed it. |
 | `waitTimeoutMillis` | How long a request can wait for a pool connection. |
 | `idleTimeoutMillis` | How quickly unused pool connections are released. |
 | `maxLifetimeMillis` | Recycling long-lived PostgreSQL connections. |

package/dist/ducklake-storage.d.ts

@@ -1,5 +1,5 @@
 import type { DatasetDefinition, DatasetRow, LakeStorageWithSql } from "@sixb/core";
-import type { BeginDatasetMergeInput, BeginDatasetWriteInput, DatasetCatalogState, DatasetVersion, LakeMergeSession, LakeWriteSession, ReadDatasetRowsInput } from "@sixb/core/lake-storage";
+import type { BeginDatasetMergeInput, BeginDatasetWriteInput, DatasetCatalogState, DatasetChanges, DatasetVersion, LakeMergeSession, LakeWriteSession, ReadDatasetChangesInput, ReadDatasetRowsInput } from "@sixb/core/lake-storage";
 import { DuckLakeSqlExecutor } from "./internal/ducklake-sql-executor";
 import type { DuckLakeMaintenanceOptions, DuckLakeMaintenanceReport, DuckLakeStorageOptions } from "./types";
 /**
@@ -19,6 +19,7 @@ export declare class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
     private readonly datasets;
     private readonly maintenance;
     private readonly rows;
+    private readonly changes;
     private readonly snapshotReader;
     private readonly writes;
     constructor(options: DuckLakeStorageOptions);
@@ -33,6 +34,7 @@ export declare class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
     getLatestVersion(datasetId: string): Promise<DatasetVersion | null>;
     getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null>;
     readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>;
+    readChanges(input: ReadDatasetChangesInput): Promise<DatasetChanges | null>;
     runMaintenance(options?: DuckLakeMaintenanceOptions): Promise<DuckLakeMaintenanceReport>;
     /**
      * `close()` is provider-specific and intentionally not part of the core

package/dist/ducklake-storage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ducklake-storage.d.ts","sourceRoot":"","sources":["../src/ducklake-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAMhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAA;AAEtE,OAAO,KAAK,EACV,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAEhB;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,kBAAkB,CAAC,QAAQ,CAAC;IAClE,QAAQ,CAAC,QAAQ;;;MAA8C;IAC/D,QAAQ,CAAC,GAAG,EAAE,mBAAmB,CAAA;IAEjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,OAAO,EAAE,sBAAsB;IAmCrC,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIxE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIrD,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;IAIpC,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAInF,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAInE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAItF,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;IAI1D,cAAc,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAI9F;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
+{"version":3,"file":"ducklake-storage.d.ts","sourceRoot":"","sources":["../src/ducklake-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAOhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAA;AAEtE,OAAO,KAAK,EACV,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAEhB;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,kBAAkB,CAAC,QAAQ,CAAC;IAClE,QAAQ,CAAC,QAAQ;;;MAA8C;IAC/D,QAAQ,CAAC,GAAG,EAAE,mBAAmB,CAAA;IAEjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,OAAO,EAAE,sBAAsB;IAwCrC,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIxE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIrD,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;IAIpC,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAInF,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAInE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAItF,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;IAIhE,WAAW,CAAC,KAAK,EAAE,uBAAuB,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAIrE,cAAc,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAI9F;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}