@sixb/ducklake 0.1.6 → 0.1.7

package/dist/index.js

@@ -771,7 +771,7 @@ function rowImage(row, prefix, columns) {
 }
 
 // src/internal/ducklake-connection-manager.ts
-import { LakeStorageError as LakeStorageError10 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError11 } from "@sixb/core/lake-storage";
 
 // src/internal/catalog-key.ts
 import { resolve } from "path";
@@ -791,9 +791,14 @@ function localCatalogCoordinationKey(options) {
 }
 
 // src/internal/duckdb-runtime.ts
+import { mkdir, mkdtemp, rm } from "fs/promises";
+import { join, resolve as resolve2 } from "path";
 import {
   DuckDBInstance
 } from "@duckdb/node-api";
+import { LakeStorageError as LakeStorageError10 } from "@sixb/core/lake-storage";
+
+// src/internal/duckdb-reader.ts
 import { LakeStorageError as LakeStorageError9 } from "@sixb/core/lake-storage";
 
 // src/internal/duckdb-value-converter.ts
@@ -816,16 +821,147 @@ var sixbDuckDbValueConverter = (value, type, converter) => {
   return JSDuckDBValueConverter(value, type, converter);
 };
 
+// src/internal/duckdb-reader.ts
+class NodeDuckDbReader {
+  connection;
+  options;
+  resultPromise;
+  pending;
+  closePromise;
+  finished = false;
+  consumed = false;
+  onAbort = () => {
+    this.close().catch(() => {});
+  };
+  constructor(connection, options) {
+    this.connection = connection;
+    this.options = options;
+  }
+  async start(sql) {
+    this.options.signal?.throwIfAborted();
+    this.options.signal?.addEventListener("abort", this.onAbort, { once: true });
+    await this.track(this.connection.run("SET preserve_insertion_order = true"));
+    this.assertOpen();
+    const prepared = await this.track(this.connection.prepare(sql));
+    try {
+      this.assertOpen();
+      this.resultPromise = this.track(prepared.startStream().getResult()).finally(() => {
+        prepared.destroySync();
+      });
+      this.resultPromise.catch(() => {});
+    } catch (error) {
+      prepared.destroySync();
+      throw error;
+    }
+  }
+  async* rows() {
+    if (this.consumed)
+      throw new LakeStorageError9("[SixbDuckLake] Reader can only be consumed once.");
+    this.consumed = true;
+    try {
+      this.assertOpen();
+      const result = await this.resultPromise;
+      this.assertOpen();
+      const batches = result.yieldConvertedRowObjects(sixbDuckDbValueConverter)[Symbol.asyncIterator]();
+      while (true) {
+        this.assertOpen();
+        const batch = await this.track(batches.next());
+        this.assertOpen();
+        if (batch.done) {
+          this.finished = true;
+          return;
+        }
+        for (const row of batch.value) {
+          this.assertOpen();
+          yield row;
+        }
+      }
+    } finally {
+      await this.close();
+    }
+  }
+  close() {
+    if (!this.closePromise) {
+      this.options.signal?.removeEventListener("abort", this.onAbort);
+      if (this.pending)
+        this.connection.interrupt();
+      this.closePromise = this.finish();
+    }
+    return this.closePromise;
+  }
+  async finish() {
+    try {
+      await this.pending?.catch(() => {});
+      if (!this.finished) {
+        await this.connection.run("SELECT 1");
+      }
+    } finally {
+      this.resultPromise = undefined;
+      try {
+        this.connection.closeSync();
+      } finally {
+        await this.options.onClose?.();
+      }
+    }
+  }
+  async track(operation) {
+    this.pending = operation;
+    try {
+      return await operation;
+    } catch (error) {
+      this.assertOpen();
+      throw error;
+    } finally {
+      if (this.pending === operation)
+        this.pending = undefined;
+    }
+  }
+  assertOpen() {
+    this.options.signal?.throwIfAborted();
+    if (this.closePromise)
+      throw new LakeStorageError9("[SixbDuckLake] Reader is closed.");
+  }
+}
+
 // src/internal/duckdb-runtime.ts
 class NodeDuckDbRuntime {
   instance;
   connection;
+  temporaryDirectory;
   closing = false;
   closed = false;
   operations = Promise.resolve();
-  constructor(instance, connection) {
+  readers = new Set;
+  closePromise;
+  constructor(instance, connection, temporaryDirectory) {
     this.instance = instance;
     this.connection = connection;
+    this.temporaryDirectory = temporaryDirectory;
+  }
+  async openReader(sql, options = {}) {
+    return this.enqueue(async () => {
+      this.assertAcceptingOperations();
+      const connection = await this.instance.connect();
+      if (this.closing) {
+        connection.closeSync();
+        this.assertAcceptingOperations();
+      }
+      const reader = new NodeDuckDbReader(connection, {
+        ...options,
+        onClose: async () => {
+          this.readers.delete(reader);
+          await options.onClose?.();
+        }
+      });
+      this.readers.add(reader);
+      try {
+        await reader.start(sql);
+        return reader;
+      } catch (error) {
+        await reader.close();
+        throw error;
+      }
+    });
   }
   async run(sql, values) {
     await this.enqueue(async () => {
@@ -845,28 +981,6 @@ class NodeDuckDbRuntime {
       return reader.convertRowObjects(sixbDuckDbValueConverter);
     });
   }
-  async* streamRows(sql, values) {
-    this.assertAcceptingOperations();
-    let releaseOperation;
-    const operationFinished = new Promise((resolve) => {
-      releaseOperation = resolve;
-    });
-    const waitForTurn = this.operations.then(() => {
-      this.assertNotClosed();
-    });
-    this.operations = waitForTurn.then(() => operationFinished).then(() => {}, () => {});
-    try {
-      await waitForTurn;
-      const result = await this.connection.stream(sql, values === undefined ? undefined : [...values]);
-      for await (const batch of result.yieldConvertedRowObjects(sixbDuckDbValueConverter)) {
-        for (const row of batch) {
-          yield row;
-        }
-      }
-    } finally {
-      releaseOperation?.();
-    }
-  }
   async withAppender(tableName, useAppender) {
     return this.enqueue(async () => {
       const appender = await this.connection.createAppender(tableName);
@@ -882,15 +996,28 @@ class NodeDuckDbRuntime {
       return useRuntime(new NodeDuckDbExclusiveRuntime(this.connection));
     });
   }
-  async close() {
-    if (this.closed) {
-      return;
-    }
+  close() {
+    this.closePromise ??= this.finish();
+    return this.closePromise;
+  }
+  async stopReaders() {
+    const results = await Promise.allSettled([...this.readers].map((reader) => reader.close()));
+    const failed = results.find((result) => result.status === "rejected");
+    if (failed?.status === "rejected")
+      throw failed.reason;
+  }
+  async finish() {
     this.closing = true;
+    const results = await Promise.allSettled([...this.readers].map((reader) => reader.close()));
     await this.operations.catch(() => {});
     this.closed = true;
     this.connection.closeSync();
     this.instance.closeSync();
+    if (this.temporaryDirectory)
+      await rm(this.temporaryDirectory, { recursive: true, force: true });
+    const failed = results.find((result) => result.status === "rejected");
+    if (failed?.status === "rejected")
+      throw failed.reason;
   }
   enqueue(operation) {
     this.assertAcceptingOperations();
@@ -903,12 +1030,12 @@ class NodeDuckDbRuntime {
   }
   assertAcceptingOperations() {
     if (this.closing || this.closed) {
-      throw new LakeStorageError9("[SixbDuckLake] DuckDB runtime is closed.");
+      throw new LakeStorageError10("[SixbDuckLake] DuckDB runtime is closed.");
     }
   }
   assertNotClosed() {
     if (this.closed) {
-      throw new LakeStorageError9("[SixbDuckLake] DuckDB runtime is closed.");
+      throw new LakeStorageError10("[SixbDuckLake] DuckDB runtime is closed.");
     }
   }
 }
@@ -968,8 +1095,27 @@ class NodeDuckDbAppenderAdapter {
   }
 }
 async function createDuckDbRuntime(options = {}) {
-  const instance = await DuckDBInstance.create(options.path ?? ":memory:", options.config ?? {});
-  return new NodeDuckDbRuntime(instance, await instance.connect());
+  const path = options.path ?? ":memory:";
+  const base = options.config?.temp_directory ?? (path === ":memory:" ? ".tmp" : `${path}.tmp`);
+  let temporaryDirectory;
+  if (base !== "") {
+    const directory = resolve2(base);
+    await mkdir(directory, { recursive: true });
+    temporaryDirectory = await mkdtemp(join(directory, "sixb-"));
+  }
+  let instance;
+  try {
+    instance = await DuckDBInstance.create(path, {
+      ...options.config,
+      ...temporaryDirectory ? { temp_directory: temporaryDirectory } : {}
+    });
+    return new NodeDuckDbRuntime(instance, await instance.connect(), temporaryDirectory);
+  } catch (error) {
+    instance?.closeSync();
+    if (temporaryDirectory)
+      await rm(temporaryDirectory, { recursive: true, force: true });
+    throw error;
+  }
 }
 async function installDuckLakeExtensions(runtime, options) {
   await runtime.run("INSTALL ducklake");
@@ -1002,6 +1148,62 @@ async function setupDuckLake(runtime, options, setupOptions = {}) {
   }
 }
 
+// src/internal/ducklake-read-gate.ts
+class DuckLakeReadGate {
+  active = 0;
+  maintenance = false;
+  waiters = new Set;
+  async acquire(signal) {
+    while (true) {
+      signal.throwIfAborted();
+      if (!this.maintenance) {
+        this.active++;
+        let released = false;
+        return () => {
+          if (released)
+            return;
+          released = true;
+          this.active--;
+          this.wake();
+        };
+      }
+      await this.wait(signal);
+    }
+  }
+  async withMaintenance(signal, run) {
+    while (this.active > 0 || this.maintenance)
+      await this.wait(signal);
+    signal.throwIfAborted();
+    this.maintenance = true;
+    try {
+      return await run();
+    } finally {
+      this.maintenance = false;
+      this.wake();
+    }
+  }
+  wait(signal) {
+    return new Promise((resolve, reject) => {
+      const wake = () => {
+        signal.removeEventListener("abort", abort);
+        this.waiters.delete(wake);
+        resolve();
+      };
+      const abort = () => {
+        this.waiters.delete(wake);
+        reject(signal.reason);
+      };
+      signal.throwIfAborted();
+      this.waiters.add(wake);
+      signal.addEventListener("abort", abort, { once: true });
+    });
+  }
+  wake() {
+    for (const wake of this.waiters)
+      wake();
+  }
+}
+
 // src/internal/ducklake-connection-manager.ts
 class DuckLakeConnectionManager {
   options;
@@ -1009,18 +1211,77 @@ class DuckLakeConnectionManager {
   installExtensionsPromise;
   attachPromise;
   refreshPromise;
-  localAttachmentLock = Promise.resolve();
+  attachmentLock = Promise.resolve();
   observedLocalCatalogGeneration;
   attached = false;
   closed = false;
+  closePromise;
   runtimePoisoned = false;
+  pinnedReaders = 0;
+  refreshRequested = false;
+  shutdown = new AbortController;
+  reads = new DuckLakeReadGate;
+  maxStreamingReads;
   constructor(options) {
     this.options = options;
     this.observedLocalCatalogGeneration = currentLocalCatalogGeneration(options);
+    this.maxStreamingReads = options.maxStreamingReads ?? 4;
+    if (!Number.isSafeInteger(this.maxStreamingReads) || this.maxStreamingReads < 1) {
+      throw new LakeStorageError11("[SixbDuckLake] maxStreamingReads must be a positive integer.");
+    }
+  }
+  async acquireRead(signal) {
+    this.assertOpen();
+    const combined = signal ? AbortSignal.any([signal, this.shutdown.signal]) : this.shutdown.signal;
+    return { signal: combined, release: await this.reads.acquire(combined) };
+  }
+  async nativeReaderLimit(runtime) {
+    if (this.options.catalog.type === "duckdb")
+      return this.maxStreamingReads;
+    if (this.options.catalog.type !== "postgres")
+      return 0;
+    const [setting] = await runtime.query("SELECT current_setting('pg_pool_max_connections') AS value");
+    const budget = Number(setting?.value);
+    return Number.isFinite(budget) ? Math.min(this.maxStreamingReads, Math.max(0, budget - 4)) : 0;
+  }
+  async openReader(runtime, sql, signal, releaseRead) {
+    const limit = await this.nativeReaderLimit(runtime);
+    signal.throwIfAborted();
+    if (this.pinnedReaders >= limit)
+      return null;
+    this.pinnedReaders++;
+    let starting = true;
+    let released = false;
+    const onClose = async () => {
+      if (released)
+        return;
+      released = true;
+      this.pinnedReaders--;
+      if (signal.aborted || this.closed || this.runtimePoisoned)
+        releaseRead();
+      if (starting || this.closed || this.runtimePoisoned)
+        return;
+      await this.withAttachmentLock(async () => {
+        if (this.pinnedReaders === 0 && this.refreshRequested)
+          await this.resetRuntimeUnlocked();
+        await this.detachDuckDbCatalogAfterLease();
+      });
+    };
+    try {
+      return await runtime.openReader(sql, { signal, onClose });
+    } catch (error) {
+      await onClose();
+      throw error;
+    } finally {
+      starting = false;
+    }
+  }
+  async withMaintenanceRuntime(run) {
+    return this.reads.withMaintenance(this.shutdown.signal, () => this.withExclusiveAttached(run));
   }
   assertOpen() {
     if (this.closed) {
-      throw new LakeStorageError10("[SixbDuckLake] DuckLakeStorage is closed.");
+      throw new LakeStorageError11("[SixbDuckLake] DuckLakeStorage is closed.");
     }
   }
   async runtime() {
@@ -1051,7 +1312,7 @@ class DuckLakeConnectionManager {
   }
   async acquireAttachedRuntime() {
     this.assertOpen();
-    const releaseLock = await this.acquireLocalAttachmentLock();
+    const releaseLock = await this.acquireAttachmentLock();
     let released = false;
     const release = async () => {
       if (released) {
@@ -1060,8 +1321,8 @@ class DuckLakeConnectionManager {
       released = true;
       try {
         if (this.runtimePoisoned) {
-          this.runtimePoisoned = false;
           await this.discardRuntimeUnlocked();
+          this.runtimePoisoned = false;
         } else {
           await this.detachDuckDbCatalogAfterLease();
         }
@@ -1110,7 +1371,7 @@ class DuckLakeConnectionManager {
     }
   }
   async resetRuntime() {
-    const releaseLock = await this.acquireLocalAttachmentLock();
+    const releaseLock = await this.acquireAttachmentLock();
     try {
       await this.resetRuntimeUnlocked();
     } finally {
@@ -1118,6 +1379,11 @@ class DuckLakeConnectionManager {
     }
   }
   async resetRuntimeUnlocked() {
+    if (this.pinnedReaders > 0) {
+      this.refreshRequested = true;
+      return;
+    }
+    this.refreshRequested = false;
     const runtimePromise = this.runtimePromise;
     if (runtimePromise === undefined) {
       return;
@@ -1158,7 +1424,7 @@ class DuckLakeConnectionManager {
     }
     if (this.refreshPromise === undefined) {
       let refreshPromise;
-      refreshPromise = this.withLocalAttachmentLock(() => this.refreshForExternalChangesUnlocked()).finally(() => {
+      refreshPromise = this.withAttachmentLock(() => this.refreshForExternalChangesUnlocked()).finally(() => {
         if (this.refreshPromise === refreshPromise) {
           this.refreshPromise = undefined;
         }
@@ -1178,8 +1444,10 @@ class DuckLakeConnectionManager {
       return;
     }
     try {
-      await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
-      this.attached = false;
+      if (this.pinnedReaders === 0) {
+        await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
+        this.attached = false;
+      }
     } catch {} finally {
       this.markLocalCatalogChanged();
     }
@@ -1195,11 +1463,13 @@ class DuckLakeConnectionManager {
   needsLocalCatalogRefresh() {
     return this.observedLocalCatalogGeneration !== currentLocalCatalogGeneration(this.options);
   }
-  async close() {
-    if (this.closed) {
-      return;
-    }
+  close() {
+    this.closePromise ??= this.finishClose();
+    return this.closePromise;
+  }
+  async finishClose() {
     this.closed = true;
+    this.shutdown.abort(new LakeStorageError11("[SixbDuckLake] DuckLakeStorage is closed."));
     const runtimePromise = this.runtimePromise;
     this.runtimePromise = undefined;
     await this.closeRuntimePromise(runtimePromise, {
@@ -1258,13 +1528,13 @@ class DuckLakeConnectionManager {
     ]);
   }
   async detachRuntime() {
-    await this.withLocalAttachmentLock(() => this.detachRuntimeUnlocked());
+    await this.withAttachmentLock(() => this.detachRuntimeUnlocked());
   }
   poisonRuntime() {
     this.runtimePoisoned = true;
   }
   async detachDuckDbCatalogAfterLease() {
-    if (this.options.catalog.type !== "duckdb" || !this.attached) {
+    if (this.options.catalog.type !== "duckdb" || !this.attached || this.pinnedReaders > 0) {
       return;
     }
     try {
@@ -1280,6 +1550,8 @@ class DuckLakeConnectionManager {
     await this.closeRuntimePromise(runtimePromise, { detach: false }).catch(() => {});
   }
   async detachRuntimeUnlocked() {
+    if (this.pinnedReaders > 0)
+      return;
     if (this.attachPromise !== undefined) {
       await this.attachPromise;
     }
@@ -1290,24 +1562,21 @@ class DuckLakeConnectionManager {
     await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
     this.attached = false;
   }
-  async withLocalAttachmentLock(run) {
-    const releaseLock = await this.acquireLocalAttachmentLock();
+  async withAttachmentLock(run) {
+    const releaseLock = await this.acquireAttachmentLock();
     try {
       return await run();
     } finally {
       releaseLock();
     }
   }
-  async acquireLocalAttachmentLock() {
-    if (this.options.catalog.type === "postgres") {
-      return () => {};
-    }
-    const previous = this.localAttachmentLock.catch(() => {});
+  async acquireAttachmentLock() {
+    const previous = this.attachmentLock.catch(() => {});
     let release;
     const current = new Promise((resolve) => {
       release = resolve;
     });
-    this.localAttachmentLock = previous.then(() => current);
+    this.attachmentLock = previous.then(() => current);
     await previous;
     let released = false;
     return () => {
@@ -1331,12 +1600,16 @@ class DuckLakeConnectionManager {
     await this.closeRuntime(runtime, options);
   }
   async closeRuntime(runtime, options) {
-    if (options.detach) {
-      try {
-        await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
-      } catch {}
+    try {
+      await runtime.stopReaders();
+      if (options.detach) {
+        try {
+          await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`);
+        } catch {}
+      }
+    } finally {
+      await runtime.close();
     }
-    await runtime.close();
   }
 }
 var localCatalogGenerations = new Map;
@@ -1358,12 +1631,12 @@ function incrementLocalCatalogGeneration(options) {
 import { randomUUID as randomUUID2 } from "crypto";
 import {
   assertDatasetDefinition,
-  LakeStorageError as LakeStorageError13,
+  LakeStorageError as LakeStorageError14,
   planDatasetDefinitionUpdate
 } from "@sixb/core/lake-storage";
 
 // src/internal/ducklake-catalog-schema.ts
-import { LakeStorageError as LakeStorageError11 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError12 } from "@sixb/core/lake-storage";
 var SIXB_PRIMARY_KEY_COMMENT_NAMESPACE = "sixb:primary-key:";
 var DATASET_SEQUENCE_COLUMN_COMMENT = "sixb.sequenceBy.v1";
 var SIXB_PRIMARY_KEY_COMMENT_PREFIX = `${SIXB_PRIMARY_KEY_COMMENT_NAMESPACE}v1:`;
@@ -1414,14 +1687,14 @@ function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName, rows) {
       throwInvalidPrimaryKeyMetadata(tableName, column, comment);
     }
     if (seenOrdinals.has(ordinal)) {
-      throw new LakeStorageError11(`[SixbDuckLake] Dataset table '${tableName}' has duplicate primary-key ordinal '${ordinal}'.`);
+      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' has duplicate primary-key ordinal '${ordinal}'.`);
     }
     seenOrdinals.add(ordinal);
     if (duckDbTypeToDatasetColumnType(column.columnType) !== "string") {
-      throw new LakeStorageError11(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must map to type 'string'.`);
+      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must map to type 'string'.`);
     }
     if (column.nullsAllowed) {
-      throw new LakeStorageError11(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must not be nullable.`);
+      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must not be nullable.`);
     }
     keyedColumns.push({ column, ordinal });
   }
@@ -1431,7 +1704,7 @@ function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName, rows) {
   keyedColumns.sort((left, right) => left.ordinal - right.ordinal);
   for (const [expectedOrdinal, keyedColumn] of keyedColumns.entries()) {
     if (keyedColumn.ordinal !== expectedOrdinal) {
-      throw new LakeStorageError11(`[SixbDuckLake] Dataset table '${tableName}' primary-key ordinals must be contiguous from 0.`);
+      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' primary-key ordinals must be contiguous from 0.`);
     }
   }
   const columnNames = keyedColumns.map(({ column }) => column.columnName);
@@ -1441,7 +1714,7 @@ function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName, rows) {
   return columnNames;
 }
 function throwInvalidPrimaryKeyMetadata(tableName, column, comment) {
-  throw new LakeStorageError11(`[SixbDuckLake] Dataset table '${tableName}' has malformed primary-key metadata '${comment}' on column '${column.columnName}'.`);
+  throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' has malformed primary-key metadata '${comment}' on column '${column.columnName}'.`);
 }
 function duckLakeCatalogColumnToDefinition(tableName, column, childrenByParentId) {
   const children = [...childrenByParentId.get(column.columnId) ?? []].sort(compareDuckLakeCatalogColumnOrder);
@@ -1458,7 +1731,7 @@ function duckLakeCatalogColumnTypeToDatasetType(tableName, column, children) {
   if (isFileRefStruct(children)) {
     return "fileRef";
   }
-  throw new LakeStorageError11(`[SixbDuckLake] DuckDB column type '${formatDuckLakeCatalogColumnType(column, children)}' in dataset table '${tableName}' cannot be mapped to a Sixb dataset column type.`);
+  throw new LakeStorageError12(`[SixbDuckLake] DuckDB column type '${formatDuckLakeCatalogColumnType(column, children)}' in dataset table '${tableName}' cannot be mapped to a Sixb dataset column type.`);
 }
 function compareDuckLakeCatalogColumnOrder(left, right) {
   return Number(left.columnOrder - right.columnOrder);
@@ -1480,7 +1753,7 @@ function formatDuckLakeCatalogColumnType(column, children) {
 }
 
 // src/internal/ducklake-current-dataset-definitions.ts
-import { LakeStorageError as LakeStorageError12 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError13 } from "@sixb/core/lake-storage";
 async function readCurrentDatasetDefinitions(input) {
   if (input.datasetIds?.length === 0) {
     return new Map;
@@ -1607,7 +1880,7 @@ async function readCurrentDatasetPartitions(input, tableNames) {
       transform: getString(row, "transform")
     };
     if (partition.transform !== "identity") {
-      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${partition.tableName}' uses unsupported DuckLake partition transform '${partition.transform}'.`);
+      throw new LakeStorageError13(`[SixbDuckLake] Dataset table '${partition.tableName}' uses unsupported DuckLake partition transform '${partition.transform}'.`);
     }
     return partition;
   });
@@ -1709,7 +1982,7 @@ class DuckLakeDatasetCatalog {
     if (failures.length > 0) {
       const details = failures.join(`
 `);
-      throw new LakeStorageError13(`[SixbLake] Lake dataset definition check failed for ${failures.length} dataset(s).
+      throw new LakeStorageError14(`[SixbLake] Lake dataset definition check failed for ${failures.length} dataset(s).
 ${details}`);
     }
   }
@@ -1745,12 +2018,12 @@ ${details}`);
     assertDuckLakeSnapshotId(snapshotId);
     const columns = await this.readDatasetColumnsAtSnapshot(runtime, tableName, tableId, snapshotId);
     if (columns.length === 0) {
-      throw new LakeStorageError13(`[SixbDuckLake] Could not reconstruct schema for dataset '${datasetId}' at DuckLake snapshot '${snapshotId}'.`);
+      throw new LakeStorageError14(`[SixbDuckLake] Could not reconstruct schema for dataset '${datasetId}' at DuckLake snapshot '${snapshotId}'.`);
     }
     return duckLakeCatalogColumnsToDatasetSchema(tableName, columns);
   }
   assertSchema(definition) {
-    assertDatasetDefinition(definition, (message) => new LakeStorageError13(`[SixbDuckLake] ${message}`));
+    assertDatasetDefinition(definition, (message) => new LakeStorageError14(`[SixbDuckLake] ${message}`));
   }
   async applyExistingDatasetUpdate(plan) {
     const tableName = encodeDatasetTableName(plan.definition.id);
@@ -1797,7 +2070,7 @@ ${details}`);
       await runtime.run(`ALTER TABLE ${qualifiedName} SET PARTITIONED BY (${definition.partitionBy.map((columnName) => quoteIdentifier(columnName)).join(", ")})`);
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
-      throw new LakeStorageError13(`[SixbDuckLake] Dataset '${definition.id}' cannot apply partitionBy because DuckLake rejected ALTER TABLE SET PARTITIONED BY: ${message}`);
+      throw new LakeStorageError14(`[SixbDuckLake] Dataset '${definition.id}' cannot apply partitionBy because DuckLake rejected ALTER TABLE SET PARTITIONED BY: ${message}`);
     }
   }
   async applyPrimaryKey(runtime, qualifiedName, definition) {
@@ -1856,12 +2129,12 @@ ${details}`);
 }
 function assertDuckLakeSnapshotId(snapshotId) {
   if (!/^\d+$/.test(snapshotId)) {
-    throw new LakeStorageError13(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
+    throw new LakeStorageError14(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
   }
 }
 
 // src/internal/ducklake-maintenance.ts
-import { LakeStorageError as LakeStorageError14 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError15 } from "@sixb/core/lake-storage";
 var DEFAULT_RETENTION = "7 days";
 
 class DuckLakeMaintenance {
@@ -1877,9 +2150,9 @@ class DuckLakeMaintenance {
     const deleteOlderThan = options.deleteOlderThan ?? expireOlderThan;
     const dryRun = options.dryRun ?? false;
     if (this.options.readOnly && !dryRun) {
-      throw new LakeStorageError14("[SixbDuckLake] DuckLake maintenance cannot run in read-only mode.");
+      throw new LakeStorageError15("[SixbDuckLake] DuckLake maintenance cannot run in read-only mode.");
     }
-    return this.connections.withExclusiveAttached(async (runtime) => {
+    return this.connections.withMaintenanceRuntime(async (runtime) => {
       const counts = await runMaintenanceProcedures(runtime, this.options, {
         expireOlderThan,
         deleteOlderThan
@@ -1914,7 +2187,7 @@ function maintenanceProcedureSql(procedure, aliasSql, olderThan, dryRun) {
 }
 
 // src/internal/ducklake-row-reader.ts
-import { LakeStorageError as LakeStorageError15 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError16 } from "@sixb/core/lake-storage";
 var READ_PAGE_ROWS = 5000;
 var PHYSICAL_ROW_ID_ALIAS = "__sixb_physical_row_id";
 
@@ -1928,58 +2201,68 @@ class DuckLakeRowReader {
     this.snapshots = snapshots;
   }
   async* readRows(input) {
-    this.connections.assertOpen();
-    const lease = await this.connections.acquireAttachedRuntime();
+    const read = await this.connections.acquireRead(input.signal);
+    let reader = null;
     try {
-      const runtime = lease.runtime;
-      const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId);
-      if (!tableRef) {
-        throw new LakeStorageError15(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`);
-      }
-      const version = await this.resolveVersion(runtime, tableRef, input.versionId);
-      const selectedColumns = this.resolveReadColumns(tableRef.datasetId, version.schema, input.columns);
-      const snapshotId = parseVersionId(version.versionId);
-      const columnsSql = selectedColumns.map((column) => quoteIdentifier(column.name)).join(", ");
-      const tableSql = qualifiedTableName(this.options, tableRef.tableName);
-      const schemaNames = new Set(version.schema.columns.map((column) => column.name.toLowerCase()));
-      const canUsePhysicalRowId = !schemaNames.has("rowid") && !schemaNames.has(PHYSICAL_ROW_ID_ALIAS.toLowerCase());
-      const orderSql = canUsePhysicalRowId ? " ORDER BY rowid" : "";
-      const baseSql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})`;
-      const requestedOffset = Math.max(0, Math.trunc(input.offset ?? 0));
-      let remaining = input.limit === undefined ? undefined : Math.max(0, Math.trunc(input.limit));
-      if (remaining === 0)
+      const prepared = await this.connections.withAttachedRuntime(async (runtime) => {
+        read.signal.throwIfAborted();
+        const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId);
+        if (!tableRef) {
+          throw new LakeStorageError16(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`);
+        }
+        const version = await this.resolveVersion(runtime, tableRef, input.versionId);
+        const columns = this.resolveReadColumns(tableRef.datasetId, version.schema, input.columns);
+        const names = new Set(version.schema.columns.map((column) => column.name.toLowerCase()));
+        const physicalOrder = !names.has("rowid") && !names.has(PHYSICAL_ROW_ID_ALIAS);
+        const relation = `${qualifiedTableName(this.options, tableRef.tableName)} AT (VERSION => ${parseVersionId(version.versionId)})`;
+        const columnsSql = columns.map((column) => quoteIdentifier(column.name)).join(", ");
+        const offset = Math.max(0, Math.trunc(input.offset ?? 0));
+        const limit = input.limit === undefined ? undefined : Math.max(0, Math.trunc(input.limit));
+        const sql = `SELECT ${columnsSql} FROM ${relation}${physicalOrder ? " ORDER BY rowid" : ""}${limit === undefined ? "" : ` LIMIT ${limit}`}${offset > 0 ? ` OFFSET ${offset}` : ""}`;
+        const cursor = limit === undefined || limit > READ_PAGE_ROWS ? await this.connections.openReader(runtime, sql, read.signal, read.release) : null;
+        const query = { columnsSql, relation, physicalOrder };
+        const firstPage = cursor || limit === 0 ? [] : await runtime.query(pageSql(query, Math.min(limit ?? READ_PAGE_ROWS, READ_PAGE_ROWS), offset, null));
+        return { columns, ...query, offset, limit, cursor, firstPage };
+      });
+      reader = prepared.cursor;
+      if (!reader)
+        read.signal.addEventListener("abort", read.release, { once: true });
+      read.signal.throwIfAborted();
+      if (reader) {
+        yield* this.normalizeRows(reader.rows(), prepared.columns, read.signal);
         return;
-      let offset = requestedOffset;
-      let physicalCursor = null;
-      while (true) {
-        const pageRows = remaining === undefined ? READ_PAGE_ROWS : Math.min(READ_PAGE_ROWS, remaining);
-        const pageSql = canUsePhysicalRowId ? `SELECT rowid AS ${quoteIdentifier(PHYSICAL_ROW_ID_ALIAS)}, ${columnsSql}
-              FROM ${tableSql} AT (VERSION => ${snapshotId})
-              ${physicalCursor === null ? "" : `WHERE rowid > ${physicalCursor}`}
-              ORDER BY rowid LIMIT ${pageRows}
-              ${physicalCursor === null && offset > 0 ? `OFFSET ${offset}` : ""}` : `${baseSql}${orderSql} LIMIT ${pageRows} OFFSET ${offset}`;
-        const rows = await runtime.query(pageSql);
-        if (rows.length === 0)
-          return;
-        yield* this.normalizeRows(rows, selectedColumns);
-        offset += rows.length;
-        if (remaining !== undefined) {
+      }
+      let offset = prepared.offset;
+      let remaining = prepared.limit;
+      let cursor = null;
+      let page = prepared.firstPage;
+      while (remaining !== 0) {
+        read.signal.throwIfAborted();
+        const pageSize = Math.min(remaining ?? READ_PAGE_ROWS, READ_PAGE_ROWS);
+        const rows = page ?? await this.connections.withAttachedRuntime((runtime) => runtime.query(pageSql(prepared, pageSize, offset, cursor)));
+        page = undefined;
+        read.signal.throwIfAborted();
+        yield* this.normalizeRows(rows, prepared.columns, read.signal);
+        if (remaining !== undefined)
           remaining -= rows.length;
-          if (remaining === 0)
-            return;
-        }
-        if (canUsePhysicalRowId) {
-          physicalCursor = getBigIntLike(rows[rows.length - 1], PHYSICAL_ROW_ID_ALIAS);
-        }
-        if (rows.length < pageRows)
+        if (rows.length < pageSize)
           return;
+        if (prepared.physicalOrder)
+          cursor = getBigIntLike(rows[rows.length - 1], PHYSICAL_ROW_ID_ALIAS);
+        offset += rows.length;
       }
     } finally {
-      await lease.release();
+      read.signal.removeEventListener("abort", read.release);
+      try {
+        await reader?.close();
+      } finally {
+        read.release();
+      }
     }
   }
-  async* normalizeRows(rows, selectedColumns) {
+  async* normalizeRows(rows, selectedColumns, signal) {
     for await (const row of rows) {
+      signal.throwIfAborted();
       const output = {};
       for (const column of selectedColumns) {
         output[column.name] = normalizeReadValue(row[column.name], column);
@@ -2010,18 +2293,23 @@ class DuckLakeRowReader {
     return columns.map((columnName) => {
       const column = columnsByName.get(columnName);
       if (!column) {
-        throw new LakeStorageError15(`[SixbDuckLake] Dataset '${datasetId}' does not have column '${columnName}' at the requested version.`);
+        throw new LakeStorageError16(`[SixbDuckLake] Dataset '${datasetId}' does not have column '${columnName}' at the requested version.`);
       }
       return column;
     });
   }
   throwNoCommittedVersion(datasetId) {
-    throw new LakeStorageError15(`[SixbDuckLake] No committed version found for dataset '${datasetId}'.`);
+    throw new LakeStorageError16(`[SixbDuckLake] No committed version found for dataset '${datasetId}'.`);
   }
 }
+function pageSql(read, limit, offset, cursor) {
+  return read.physicalOrder ? `SELECT rowid AS ${quoteIdentifier(PHYSICAL_ROW_ID_ALIAS)}, ${read.columnsSql}
+        FROM ${read.relation} ${cursor === null ? "" : `WHERE rowid > ${cursor}`}
+        ORDER BY rowid LIMIT ${limit} ${cursor === null && offset > 0 ? `OFFSET ${offset}` : ""}` : `SELECT ${read.columnsSql} FROM ${read.relation} LIMIT ${limit} OFFSET ${offset}`;
+}
 
 // src/internal/ducklake-snapshot-reader.ts
-import { LakeStorageError as LakeStorageError16 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError17 } from "@sixb/core/lake-storage";
 var SNAPSHOT_ROW_BATCH_SIZE = 128;
 var CATALOG_SNAPSHOT_SCAN_LIMIT = 512;
 
@@ -2234,7 +2522,7 @@ class DuckLakeSnapshotReader {
   }
   assertNoMetadataConflict(snapshotId, datasetId, metadata) {
     if (metadata !== undefined && metadata.datasetId !== datasetId) {
-      throw new LakeStorageError16(`[SixbDuckLake] DuckLake snapshot '${snapshotId}' changed dataset '${datasetId}' but Sixb commit metadata references dataset '${metadata.datasetId}'.`);
+      throw new LakeStorageError17(`[SixbDuckLake] DuckLake snapshot '${snapshotId}' changed dataset '${datasetId}' but Sixb commit metadata references dataset '${metadata.datasetId}'.`);
     }
   }
   async getLatestVersionForDefinition(runtime, dataset) {
@@ -2379,7 +2667,10 @@ class DuckLakeSnapshotReader {
     if (input.limit !== undefined && input.limit <= 0) {
       return [];
     }
-    const where = [];
+    const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table");
+    const where = [
+      `snapshot.snapshot_id >= (SELECT min(begin_snapshot) FROM ${ducklakeTable} WHERE table_id = ${input.tableId})`
+    ];
     if (input.exactSnapshotId !== undefined) {
       assertDuckLakeSnapshotId2(input.exactSnapshotId);
       where.push(`snapshot.snapshot_id = ${input.exactSnapshotId}`);
@@ -2388,7 +2679,7 @@ class DuckLakeSnapshotReader {
       assertDuckLakeSnapshotId2(input.beforeSnapshotId);
       where.push(`snapshot.snapshot_id < ${input.beforeSnapshotId}`);
     }
-    const whereSql = where.length === 0 ? "" : `WHERE ${where.join(" AND ")}`;
+    const whereSql = `WHERE ${where.join(" AND ")}`;
     const limitSql = input.limit === undefined ? "" : `LIMIT ${Math.max(0, Math.trunc(input.limit))}`;
     const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot");
     const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
@@ -2514,7 +2805,7 @@ class DuckLakeSnapshotReader {
 }
 function assertDuckLakeSnapshotId2(snapshotId) {
   if (!/^\d+$/.test(snapshotId)) {
-    throw new LakeStorageError16(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
+    throw new LakeStorageError17(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
   }
 }
 function fileChangeKey(snapshotId, tableId) {
@@ -2524,17 +2815,17 @@ function fileChangeKey(snapshotId, tableId) {
 // src/internal/ducklake-sql-executor.ts
 import { randomUUID as randomUUID4 } from "crypto";
 import {
-  LakeStorageError as LakeStorageError19
+  LakeStorageError as LakeStorageError20
 } from "@sixb/core/lake-storage";
 
 // src/internal/dataset-row-commit.ts
 import {
   getDatasetPrimaryKeyColumns,
-  LakeStorageError as LakeStorageError17
+  LakeStorageError as LakeStorageError18
 } from "@sixb/core/lake-storage";
 function assertDatasetWriteMode(mode, operation) {
   if (mode !== "snapshot" && mode !== "append") {
-    throw new LakeStorageError17(`[SixbDuckLake] Invalid ${operation} mode '${String(mode)}'. Expected 'snapshot' or 'append'.`);
+    throw new LakeStorageError18(`[SixbDuckLake] Invalid ${operation} mode '${String(mode)}'. Expected 'snapshot' or 'append'.`);
   }
 }
 async function applyDatasetRowsFromRelation(input) {
@@ -2594,7 +2885,7 @@ async function applyDatasetRowsFromRelation(input) {
 async function inspectUniqueKeyedRelation(input) {
   const primaryKeyColumns = getDatasetPrimaryKeyColumns(input.dataset);
   if (primaryKeyColumns === null) {
-    throw new LakeStorageError17(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before rows can be keyed.`);
+    throw new LakeStorageError18(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before rows can be keyed.`);
   }
   const keysSql = primaryKeyColumns.map((column) => quoteIdentifier(column)).join(", ");
   const [row] = await input.runtime.query(`
@@ -2609,7 +2900,7 @@ async function inspectUniqueKeyedRelation(input) {
   const rowCount = row === undefined ? 0 : Number(getBigIntLike(row, "row_count"));
   const maxKeyCount = row === undefined ? 0 : Number(getBigIntLike(row, "max_key_count"));
   if (maxKeyCount > 1) {
-    throw new LakeStorageError17(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
+    throw new LakeStorageError18(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
   }
   return rowCount;
 }
@@ -2626,7 +2917,7 @@ async function assertNoPrimaryKeyOverlap(input) {
     LIMIT 1
   `);
   if (collision !== undefined) {
-    throw new LakeStorageError17(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
+    throw new LakeStorageError18(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
   }
 }
 async function fingerprintRelation(runtime, relationSql, columnsSql) {
@@ -2660,7 +2951,7 @@ async function countRows(runtime, relationSql) {
 // src/internal/sql-transform-relations.ts
 import { randomUUID as randomUUID3 } from "crypto";
 import {
-  LakeStorageError as LakeStorageError18
+  LakeStorageError as LakeStorageError19
 } from "@sixb/core/lake-storage";
 var RELATION_PLACEHOLDER_PREFIX = "__sixb_sql_transform_relation_";
 var RELATION_PLACEHOLDER_PATTERN = /__sixb_sql_transform_relation_[0-9a-f]{32}_[0-9]+__/g;
@@ -2678,7 +2969,7 @@ function renderDuckLakeSqlTransformSql(input) {
   const context = createDuckLakeSqlTransformRelationContext(input);
   const sql = input.sql(context.relations);
   if (typeof sql !== "string") {
-    throw new LakeStorageError18("[SixbDuckLake] SQL transform body must return a SQL string.");
+    throw new LakeStorageError19("[SixbDuckLake] SQL transform body must return a SQL string.");
   }
   return context.render(sql);
 }
@@ -2716,7 +3007,7 @@ function renderDuckLakeSourceRelation(options, source) {
   return `(SELECT * FROM ${qualifiedTableName(options, tableName)} AT (VERSION => ${snapshotId}))`;
 }
 function throwUnresolvedPlaceholder() {
-  throw new LakeStorageError18("[SixbDuckLake] SQL transform contains an unresolved relation placeholder.");
+  throw new LakeStorageError19("[SixbDuckLake] SQL transform contains an unresolved relation placeholder.");
 }
 
 // src/internal/ducklake-sql-executor.ts
@@ -2810,14 +3101,14 @@ class DuckLakeSqlExecutor {
   async resolveSourceDataset(runtime, dataset) {
     const definition = await this.datasets.getDatasetOnRuntime(runtime, dataset.id);
     if (!definition) {
-      throw new LakeStorageError19(`[SixbDuckLake] Unknown SQL transform source dataset '${dataset.id}'.`);
+      throw new LakeStorageError20(`[SixbDuckLake] Unknown SQL transform source dataset '${dataset.id}'.`);
     }
     return definition;
   }
   async resolveTargetDataset(runtime, dataset) {
     const definition = await this.datasets.getDatasetOnRuntime(runtime, dataset.id);
     if (!definition) {
-      throw new LakeStorageError19(`[SixbDuckLake] Unknown SQL transform target dataset '${dataset.id}'.`);
+      throw new LakeStorageError20(`[SixbDuckLake] Unknown SQL transform target dataset '${dataset.id}'.`);
     }
     this.datasets.assertSchema(definition);
     return definition;
@@ -2938,10 +3229,10 @@ function normalizePreviewValue(value) {
   return value;
 }
 function throwNoCommittedSourceVersion(datasetId) {
-  throw new LakeStorageError19(`[SixbDuckLake] No committed version found for SQL transform source dataset '${datasetId}'.`);
+  throw new LakeStorageError20(`[SixbDuckLake] No committed version found for SQL transform source dataset '${datasetId}'.`);
 }
 function throwResultSchemaMismatch(datasetId, detail) {
-  throw new LakeStorageError19(`[SixbDuckLake] SQL transform result schema does not match target dataset '${datasetId}': ${detail}.`);
+  throw new LakeStorageError20(`[SixbDuckLake] SQL transform result schema does not match target dataset '${datasetId}': ${detail}.`);
 }
 
 // src/internal/ducklake-write-coordinator.ts
@@ -2951,20 +3242,20 @@ import {
   getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns4,
   hasDatasetInputChanges,
   LakeConcurrencyError,
-  LakeStorageError as LakeStorageError23,
+  LakeStorageError as LakeStorageError24,
   retryDatasetMergeCommit
 } from "@sixb/core/lake-storage";
 
 // src/internal/dataset-row-merge.ts
 import {
   getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns2,
-  LakeStorageError as LakeStorageError20,
+  LakeStorageError as LakeStorageError21,
   reconcileDatasetSequences
 } from "@sixb/core/lake-storage";
 async function applyDatasetMergeFromRelation(input) {
   const primaryKeyColumns = getDatasetPrimaryKeyColumns2(input.dataset);
   if (primaryKeyColumns === null) {
-    throw new LakeStorageError20(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before it can be merged.`);
+    throw new LakeStorageError21(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before it can be merged.`);
   }
   const stagingTable = quoteIdentifier(input.stagingTableName);
   const targetTable = qualifiedTableName(input.options, encodeDatasetTableName(input.dataset.id));
@@ -3125,7 +3416,7 @@ function finalChangesSelectSql(input, primaryKeyColumns) {
 async function countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns) {
   const matchColumn = primaryKeyColumns[0];
   if (matchColumn === undefined) {
-    throw new LakeStorageError20(`[SixbDuckLake] Dataset '${input.dataset.id}' primaryKey must contain at least one column.`);
+    throw new LakeStorageError21(`[SixbDuckLake] Dataset '${input.dataset.id}' primaryKey must contain at least one column.`);
   }
   const keyMatchSql = primaryKeyMatchSql(primaryKeyColumns, "target", "source");
   const targetMatch = `target.${quoteIdentifier(matchColumn)} IS NOT NULL`;
@@ -3179,7 +3470,7 @@ import {
   cloneDatasetMergeChange,
   datasetSequenceChange,
   getDatasetMergeChangeValidationError,
-  LakeStorageError as LakeStorageError21
+  LakeStorageError as LakeStorageError22
 } from "@sixb/core/lake-storage";
 
 // src/internal/row-appender.ts
@@ -3280,7 +3571,7 @@ class DuckLakeMergeSession {
     for await (const change of changes) {
       const validationError = getDatasetMergeChangeValidationError(change, this.input.dataset);
       if (validationError) {
-        throw new LakeStorageError21(`[SixbDuckLake] ${validationError}`);
+        throw new LakeStorageError22(`[SixbDuckLake] ${validationError}`);
       }
       batch.push({
         sequence: this.nextSequence,
@@ -3343,7 +3634,7 @@ class DuckLakeMergeSession {
   }
   assertOpen() {
     if (this.closed) {
-      throw new LakeStorageError21("[SixbDuckLake] Merge session is already closed.");
+      throw new LakeStorageError22("[SixbDuckLake] Merge session is already closed.");
     }
   }
   async cleanup() {
@@ -3363,7 +3654,7 @@ import { getDatasetRowValidationError } from "@sixb/core";
 import {
   encodeDatasetPrimaryKey,
   getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns3,
-  LakeStorageError as LakeStorageError22
+  LakeStorageError as LakeStorageError23
 } from "@sixb/core/lake-storage";
 var WRITE_ROW_BATCH_SIZE = 1000;
 async function createDuckLakeWriteSession(input) {
@@ -3394,13 +3685,13 @@ class DuckLakeWriteSession {
     for await (const row of rows) {
       const validationError = getDatasetRowValidationError(row, this.input.dataset);
       if (validationError) {
-        throw new LakeStorageError22(`[SixbDuckLake] ${validationError}`);
+        throw new LakeStorageError23(`[SixbDuckLake] ${validationError}`);
       }
       const clonedRow = structuredClone(row);
       if (getDatasetPrimaryKeyColumns3(this.input.dataset) !== null) {
         const primaryKey = encodeDatasetPrimaryKey(this.input.dataset, clonedRow);
         if (this.stagedPrimaryKeys.has(primaryKey) || batchPrimaryKeys.has(primaryKey)) {
-          throw new LakeStorageError22(`[SixbDuckLake] Dataset '${this.input.dataset.id}' ${this.input.mode ?? "snapshot"} source contains duplicate primary key.`);
+          throw new LakeStorageError23(`[SixbDuckLake] Dataset '${this.input.dataset.id}' ${this.input.mode ?? "snapshot"} source contains duplicate primary key.`);
         }
         batchPrimaryKeys.add(primaryKey);
       }
@@ -3457,7 +3748,7 @@ class DuckLakeWriteSession {
   }
   assertOpen() {
     if (this.closed) {
-      throw new LakeStorageError22("[SixbDuckLake] Write session is already closed.");
+      throw new LakeStorageError23("[SixbDuckLake] Write session is already closed.");
     }
   }
   async cleanup() {
@@ -3465,6 +3756,7 @@ class DuckLakeWriteSession {
       return;
     }
     this.cleanedUp = true;
+    this.stagedPrimaryKeys.clear();
     await this.dropStagingTable();
   }
   async dropStagingTable() {
@@ -3490,7 +3782,7 @@ class DuckLakeWriteCoordinator {
     this.connections.assertOpen();
     const definition = await this.datasets.getDataset(input.dataset.id);
     if (!definition) {
-      throw new LakeStorageError23(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
+      throw new LakeStorageError24(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
     }
     this.datasets.assertSchema(definition);
     assertUnsequencedDatasetWrite(definition);
@@ -3511,10 +3803,10 @@ class DuckLakeWriteCoordinator {
     this.connections.assertOpen();
     const definition = await this.datasets.getDataset(input.dataset.id);
     if (!definition) {
-      throw new LakeStorageError23(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
+      throw new LakeStorageError24(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
     }
     if (getDatasetPrimaryKeyColumns4(definition) === null) {
-      throw new LakeStorageError23(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
+      throw new LakeStorageError24(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
     }
     this.datasets.assertSchema(definition);
     const latestVersion = await this.connections.withAttachedRuntime((runtime) => this.snapshots.getLatestVersionForDefinition(runtime, definition));
@@ -3532,7 +3824,7 @@ class DuckLakeWriteCoordinator {
   async commitWrite(input) {
     const definition = await this.datasets.getDataset(input.write.dataset.id);
     if (!definition) {
-      throw new LakeStorageError23(`[SixbDuckLake] Unknown dataset '${input.write.dataset.id}'.`);
+      throw new LakeStorageError24(`[SixbDuckLake] Unknown dataset '${input.write.dataset.id}'.`);
     }
     const mode = input.write.mode ?? "snapshot";
     assertDatasetWriteMode(mode, "write");
@@ -3553,10 +3845,10 @@ class DuckLakeWriteCoordinator {
   async commitMerge(input) {
     const definition = await this.datasets.getDataset(input.merge.dataset.id);
     if (!definition) {
-      throw new LakeStorageError23(`[SixbDuckLake] Unknown dataset '${input.merge.dataset.id}'.`);
+      throw new LakeStorageError24(`[SixbDuckLake] Unknown dataset '${input.merge.dataset.id}'.`);
     }
     if (getDatasetPrimaryKeyColumns4(definition) === null) {
-      throw new LakeStorageError23(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
+      throw new LakeStorageError24(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
     }
     return retryDatasetMergeCommit(input.merge, input.commit, (rebase) => this.withCommitRuntime(async (runtime) => {
       input.commit?.signal?.throwIfAborted();
@@ -3584,7 +3876,7 @@ class DuckLakeWriteCoordinator {
             validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns
           });
           if (result.sourceRowCount !== input.changesWritten) {
-            throw new LakeStorageError23(`[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`);
+            throw new LakeStorageError24(`[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`);
           }
           return result;
         }
@@ -3606,7 +3898,7 @@ class DuckLakeWriteCoordinator {
       disableRetries: input.mode === "append" && getDatasetPrimaryKeyColumns4(input.dataset) !== null
     });
     if (result.version === null) {
-      throw new LakeStorageError23(`[SixbDuckLake] No DuckLake changes were committed for dataset '${input.dataset.id}', and no previous version exists.`);
+      throw new LakeStorageError24(`[SixbDuckLake] No DuckLake changes were committed for dataset '${input.dataset.id}', and no previous version exists.`);
     }
     return { ...result.version, outcome: result.outcome };
   }
@@ -3637,10 +3929,9 @@ class DuckLakeWriteCoordinator {
   }
   async commitDatasetVersionUnlocked(input) {
     const previousWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
-    await input.runtime.run("BEGIN TRANSACTION");
     let committed = false;
     try {
-      const latestVersion = await this.latestVersionForCommit(input);
+      const latestVersion = await this.beginVersionTransaction(input);
       if (input.expectedLatestVersionId !== undefined) {
         this.assertExpectedLatestVersion({
           datasetId: input.dataset.id,
@@ -3688,7 +3979,7 @@ class DuckLakeWriteCoordinator {
       if (version) {
         return { outcome: "created", version };
       }
-      throw new LakeStorageError23(`[SixbDuckLake] DuckLake committed snapshot '${ownSnapshotId}' for dataset '${input.dataset.id}', but Sixb could not hydrate it as a dataset version.`);
+      throw new LakeStorageError24(`[SixbDuckLake] DuckLake committed snapshot '${ownSnapshotId}' for dataset '${input.dataset.id}', but Sixb could not hydrate it as a dataset version.`);
     }
     if (!input.changeResult.dataChangeExpected) {
       const version = await this.versionForNoOpCommit(input.runtime, input.dataset, input.allowInitialNoOp ?? false);
@@ -3706,9 +3997,9 @@ class DuckLakeWriteCoordinator {
       };
     }
     if (input.committedWriteSnapshotId === input.previousWriteSnapshotId) {
-      throw new LakeStorageError23(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed without producing a snapshot for this non-empty write.`);
+      throw new LakeStorageError24(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed without producing a snapshot for this non-empty write.`);
     }
-    throw new LakeStorageError23(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed, but Sixb could not find a matching snapshot for this write.`);
+    throw new LakeStorageError24(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed, but Sixb could not find a matching snapshot for this write.`);
   }
   async applyStagedRows(input) {
     const mode = input.write.mode ?? "snapshot";
@@ -3721,7 +4012,7 @@ class DuckLakeWriteCoordinator {
       previousRowCount: input.previousRowCount
     });
     if (result.sourceRowCount !== input.rowsWritten) {
-      throw new LakeStorageError23(`[SixbDuckLake] Staged write for dataset '${input.write.dataset.id}' accepted ${input.rowsWritten} row(s), but DuckLake saw ${result.sourceRowCount} source row(s) at commit time.`);
+      throw new LakeStorageError24(`[SixbDuckLake] Staged write for dataset '${input.write.dataset.id}' accepted ${input.rowsWritten} row(s), but DuckLake saw ${result.sourceRowCount} source row(s) at commit time.`);
     }
     return result;
   }
@@ -3743,7 +4034,7 @@ class DuckLakeWriteCoordinator {
     if (changeResult.dataChangeExpected) {
       return;
     }
-    const latestVersion = knownLatestVersion ?? await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
+    const latestVersion = knownLatestVersion === undefined ? await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset) : knownLatestVersion;
     if (latestVersion ? input.mode === "merge" || !hasDatasetInputChanges(latestVersion.inputs, input.inputs) : input.allowInitialNoOp) {
       return;
     }
@@ -3759,18 +4050,39 @@ class DuckLakeWriteCoordinator {
     if (allowInitialNoOp) {
       return null;
     }
-    throw new LakeStorageError23(`[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`);
+    throw new LakeStorageError24(`[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`);
   }
   async rollbackTransaction(runtime) {
     try {
       await runtime.run("ROLLBACK");
     } catch {}
   }
-  async latestVersionForCommit(input) {
-    if (input.expectedLatestVersionId === undefined && input.mode !== "append") {
-      return null;
+  async beginVersionTransaction(input) {
+    if (!isSqliteCatalog(this.options.catalog)) {
+      await input.runtime.run("BEGIN TRANSACTION");
+      if (input.expectedLatestVersionId === undefined && input.mode !== "append") {
+        return;
+      }
+      return this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
     }
-    return this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
+    for (let attempt = 0;attempt < 3; attempt++) {
+      input.signal?.throwIfAborted();
+      const snapshotId = await this.currentSnapshotId(input.runtime);
+      const latestVersion = await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
+      await input.runtime.run("BEGIN TRANSACTION");
+      if (await this.currentSnapshotId(input.runtime) === snapshotId) {
+        return latestVersion;
+      }
+      await input.runtime.run("ROLLBACK");
+    }
+    throw new LakeConcurrencyError(`[SixbDuckLake] Catalog changed during three write preparations for dataset '${input.dataset.id}'. Retry the write.`);
+  }
+  async currentSnapshotId(runtime) {
+    const [row] = await runtime.query(`SELECT id FROM ${quoteIdentifier(duckLakeAlias(this.options))}.current_snapshot()`);
+    if (row === undefined) {
+      throw new LakeStorageError24("[SixbDuckLake] DuckLake did not return a current snapshot.");
+    }
+    return String(getBigIntLike(row, "id"));
   }
   async lastCommittedSnapshotId(runtime) {
     const [row] = await runtime.query(`SELECT id FROM ${quoteIdentifier(duckLakeAlias(this.options))}.last_committed_snapshot()`);
@@ -3819,6 +4131,9 @@ class DuckLakeWriteCoordinator {
     }
   }
 }
+function isSqliteCatalog(catalog) {
+  return catalog.type === "sqlite" || catalog.type === "custom" && catalog.uri.toLowerCase().startsWith("sqlite:");
+}
 function commitRowCount(input, changeResult, latestVersion) {
   if (input.mode === "append" && input.expectedLatestVersionId === undefined) {
     return { kind: "unknown" };
@@ -3833,7 +4148,7 @@ function commitRowCount(input, changeResult, latestVersion) {
 }
 function assertDuckLakeSnapshotId3(snapshotId) {
   if (!/^\d+$/.test(snapshotId)) {
-    throw new LakeStorageError23(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
+    throw new LakeStorageError24(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
   }
 }
 var localCatalogCommitLocks = new Map;
@@ -3939,4 +4254,4 @@ export {
   DuckLakeStorage
 };
 
-//# debugId=555594FFA13F16F264756E2164756E21
+//# debugId=35ABBEC5A94F36A964756E2164756E21