@sixb/ducklake 0.1.1 → 0.1.3

package/dist/internal/ducklake-row-reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ducklake-row-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-row-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,UAAU,EAAE,MAAM,YAAY,CAAA;AACrE,OAAO,KAAK,EAAkB,oBAAoB,EAAE,MAAM,yBAAyB,CAAA;AAEnF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAE9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAKxE;;;;;;GAMG;AACH,qBAAa,iBAAiB;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAFT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,SAAS,EAAE,sBAAsB;IAG7C,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;YAoDxD,aAAa;YAad,cAAc;IAuB5B,OAAO,CAAC,kBAAkB;IAuB1B,OAAO,CAAC,uBAAuB;CAKhC"}
+{"version":3,"file":"ducklake-row-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-row-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,UAAU,EAAE,MAAM,YAAY,CAAA;AACrE,OAAO,KAAK,EAAkB,oBAAoB,EAAE,MAAM,yBAAyB,CAAA;AAEnF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAGtD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAE9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAYxE;;;;;;GAMG;AACH,qBAAa,iBAAiB;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAFT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,SAAS,EAAE,sBAAsB;IAG7C,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;YAoExD,aAAa;YAad,cAAc;IAuB5B,OAAO,CAAC,kBAAkB;IAuB1B,OAAO,CAAC,uBAAuB;CAKhC"}

package/dist/internal/ducklake-sql-executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ducklake-sql-executor.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-sql-executor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8C,UAAU,EAAE,MAAM,YAAY,CAAA;AACxF,OAAO,EAEL,KAAK,wBAAwB,EAE7B,KAAK,wBAAwB,EAC7B,KAAK,eAAe,EAEpB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAA;AAChC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAQtD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAyB5E;;GAEG;AACH,qBAAa,mBAAoB,YAAW,eAAe,CAAC,QAAQ,CAAC;IASjE,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAZzB,QAAQ,CAAC,OAAO,EAAG,QAAQ,CAAS;IACpC,QAAQ,CAAC,YAAY;;;;MAIpB;gBAGkB,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,sBAAsB,EACjC,MAAM,EAAE,wBAAwB;IAG5C,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC;IAkB9E,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,wBAAwB,CAAC;YAkC7E,cAAc;YA2Bd,oBAAoB;YAcpB,oBAAoB;YAepB,oBAAoB;YAuBpB,+BAA+B;YAsC/B,iBAAiB;CA+BhC"}
+{"version":3,"file":"ducklake-sql-executor.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-sql-executor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8C,UAAU,EAAE,MAAM,YAAY,CAAA;AACxF,OAAO,EAEL,KAAK,wBAAwB,EAE7B,KAAK,wBAAwB,EAC7B,KAAK,eAAe,EAEpB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAA;AAChC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAStD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAoC5E;;GAEG;AACH,qBAAa,mBAAoB,YAAW,eAAe,CAAC,QAAQ,CAAC;IASjE,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAZzB,QAAQ,CAAC,OAAO,EAAG,QAAQ,CAAS;IACpC,QAAQ,CAAC,YAAY;;;;MAIpB;gBAGkB,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,sBAAsB,EACjC,MAAM,EAAE,wBAAwB;IAG5C,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC;IAkB9E,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,wBAAwB,CAAC;YAmC7E,cAAc;YA2Bd,oBAAoB;YAcpB,oBAAoB;YAepB,oBAAoB;YAuBpB,+BAA+B;YAoD/B,iBAAiB;CAuChC"}

package/dist/internal/ducklake-write-coordinator.d.ts

@@ -46,6 +46,7 @@ export declare class DuckLakeWriteCoordinator {
     private versionForCommittedWrite;
     private applyStagedRows;
     private setCommitMetadata;
+    private ensureInitialNoOpHasSnapshot;
     private versionForNoOpCommit;
     private rollbackTransaction;
     private latestVersionForCommit;

package/dist/internal/ducklake-write-coordinator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ducklake-write-coordinator.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-write-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EAGtB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,EACL,KAAK,sBAAsB,EAI5B,MAAM,sBAAsB,CAAA;AAG7B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAExE,OAAO,KAAK,EAAE,sBAAsB,EAA0B,MAAM,4BAA4B,CAAA;AAKhG,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;IAC/B,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAA;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,sBAAsB,CAAC,UAAU,CAAC,CAAA;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAA;IAClD,YAAY,CACV,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,sBAAsB,CAAC,CAAA;CACnC;AAeD,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAED;;;;;;;GAOG;AACH,qBAAa,wBAAwB;IAEjC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAHT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,sBAAsB;IAG9C,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4BpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAkC5D,WAAW;YAyBX,WAAW;IA4CnB,oBAAoB,CACxB,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;IAM9B,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAQlF,sCAAsC,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;YAiBtB,sCAAsC;YAUtC,uBAAuB;YA2CvB,4BAA4B;YA0D5B,wBAAwB;YAiExB,eAAe;YAyBf,iBAAiB;YAkCjB,oBAAoB;YAmBpB,mBAAmB;YASnB,sBAAsB;YAUtB,uBAAuB;YAYvB,wBAAwB;YAiCxB,iBAAiB;IA0B/B,OAAO,CAAC,2BAA2B;CAgBpC"}
+{"version":3,"file":"ducklake-write-coordinator.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-write-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EAGtB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,EACL,KAAK,sBAAsB,EAI5B,MAAM,sBAAsB,CAAA;AAG7B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAExE,OAAO,KAAK,EAAE,sBAAsB,EAA0B,MAAM,4BAA4B,CAAA;AAYhG,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;IAC/B,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAA;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,sBAAsB,CAAC,UAAU,CAAC,CAAA;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAA;IAClD,YAAY,CACV,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,sBAAsB,CAAC,CAAA;CACnC;AAeD,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAED;;;;;;;GAOG;AACH,qBAAa,wBAAwB;IAEjC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAHT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,sBAAsB;IAG9C,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4BpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAkC5D,WAAW;YAyBX,WAAW;IA4CnB,oBAAoB,CACxB,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;IAM9B,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAQlF,sCAAsC,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;YAiBtB,sCAAsC;YAUtC,uBAAuB;YA2CvB,4BAA4B;YA2D5B,wBAAwB;YAiExB,eAAe;YAyBf,iBAAiB;YAkCjB,4BAA4B;YA0B5B,oBAAoB;YAmBpB,mBAAmB;YASnB,sBAAsB;YAUtB,uBAAuB;YAYvB,wBAAwB;YAiCxB,iBAAiB;IA0B/B,OAAO,CAAC,2BAA2B;CAgBpC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sixb/ducklake",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "DuckDB/DuckLake lake storage provider for Sixb",
   "keywords": [
     "sixb",
@@ -44,10 +44,10 @@
     "@duckdb/node-api": "^1.5.2-r.1"
   },
   "peerDependencies": {
-    "@sixb/core": "^0.1.1"
+    "@sixb/core": "^0.1.4"
   },
   "devDependencies": {
-    "@sixb/core": "0.1.1",
+    "@sixb/core": "0.1.4",
     "@types/bun": "latest",
     "typescript": "^5.7.0"
   },

package/src/internal/ducklake-merge-session.ts

@@ -108,6 +108,7 @@ class DuckLakeMergeSession implements LakeMergeSession {
       if (batch.length >= MERGE_CHANGE_BATCH_SIZE) {
         await this.appendBatchToStagingTable(batch)
         batch = []
+        await Bun.sleep(0)
       }
     }
 

package/src/internal/ducklake-row-reader.ts

@@ -2,6 +2,7 @@ import type { DatasetColumnDefinition, DatasetRow } from "@sixb/core"
 import type { DatasetVersion, ReadDatasetRowsInput } from "@sixb/core/lake-storage"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
+import { getBigIntLike } from "./duckdb-row"
 import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
 import { type DatasetTableRef, resolveDatasetTableRef } from "./ducklake-dataset-table-ref"
@@ -10,6 +11,13 @@ import { normalizeReadValue } from "./schema"
 import { qualifiedTableName, quoteIdentifier } from "./sql"
 import { parseVersionId } from "./versions"
 
+// Large reads release the provider's single DuckDB queue slot between bounded pages. Besides
+// bounding converted-row memory, this lets a JavaScript pipeline feed one DuckLake dataset into
+// another: the destination appender can use the runtime after each page instead of waiting forever
+// behind its own still-open source stream.
+const READ_PAGE_ROWS = 5_000
+const PHYSICAL_ROW_ID_ALIAS = "__sixb_physical_row_id"
+
 /**
  * Reads dataset rows from DuckLake snapshots.
  *
@@ -27,50 +35,66 @@ export class DuckLakeRowReader {
   async *readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow> {
     this.connections.assertOpen()
 
+    // Keep the attachment lease across pages so local catalogs do not pay DETACH/ATTACH for every
+    // page. Individual query() calls still release the DuckDB queue before rows are yielded.
     const lease = await this.connections.acquireAttachedRuntime()
     try {
       const runtime = lease.runtime
-
-      // Step 1: resolve the physical table from DuckLake metadata directly. Row
-      // previews must not go through broad catalog introspection.
+      // Resolve latest exactly once so a paged read stays pinned even if another writer commits
+      // between pages. DuckLake snapshots are immutable; every page names this snapshot.
       const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId)
       if (!tableRef) {
         throw new LakeStorageError(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`)
       }
-
       const version = await this.resolveVersion(runtime, tableRef, input.versionId)
-      const snapshotId = parseVersionId(version.versionId)
       const selectedColumns = this.resolveReadColumns(
         tableRef.datasetId,
         version.schema,
         input.columns
       )
-
-      // Step 2: query DuckLake at the exact snapshot id. Version ids are just
-      // `ducklake:<snapshot_id>`, so reads can use native DuckLake time travel
-      // directly after validation.
+      const snapshotId = parseVersionId(version.versionId)
       const columnsSql = selectedColumns.map((column) => quoteIdentifier(column.name)).join(", ")
       const tableSql = qualifiedTableName(this.options, tableRef.tableName)
       // A user column named `rowid` shadows DuckDB's virtual row id. In that edge case the
       // connection-level invariant still preserves insertion order; otherwise make it explicit.
-      const orderSql = version.schema.columns.some(
-        (column) => column.name.toLowerCase() === "rowid"
-      )
-        ? ""
-        : " ORDER BY rowid"
-      const limitSql =
-        input.limit === undefined ? "" : ` LIMIT ${Math.max(0, Math.trunc(input.limit))}`
-      const offsetSql =
-        input.offset === undefined ? "" : ` OFFSET ${Math.max(0, Math.trunc(input.offset))}`
-      const sql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})${orderSql}${limitSql}${offsetSql}`
-
-      // HTTP previews are bounded, so materialize them eagerly and release the
-      // runtime queue as soon as DuckDB has returned the small page.
-      const rows = input.limit === undefined ? runtime.streamRows(sql) : await runtime.query(sql)
-
-      // Step 3: DuckDB returns driver-native values. Normalize each projected
-      // column through the schema that was active at the resolved version.
-      yield* this.normalizeRows(rows, selectedColumns)
+      const schemaNames = new Set(version.schema.columns.map((column) => column.name.toLowerCase()))
+      const canUsePhysicalRowId =
+        !schemaNames.has("rowid") && !schemaNames.has(PHYSICAL_ROW_ID_ALIAS.toLowerCase())
+      const orderSql = canUsePhysicalRowId ? " ORDER BY rowid" : ""
+      const baseSql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})`
+      const requestedOffset = Math.max(0, Math.trunc(input.offset ?? 0))
+      let remaining = input.limit === undefined ? undefined : Math.max(0, Math.trunc(input.limit))
+      if (remaining === 0) return
+
+      // Materialize one bounded page and release its queue operation before yielding it. A
+      // consumer may enqueue another DuckLake operation while processing the yielded rows.
+      // Holding a native stream open here would make that operation wait behind itself.
+      let offset = requestedOffset
+      let physicalCursor: bigint | null = null
+      while (true) {
+        const pageRows =
+          remaining === undefined ? READ_PAGE_ROWS : Math.min(READ_PAGE_ROWS, remaining)
+        const pageSql = canUsePhysicalRowId
+          ? `SELECT rowid AS ${quoteIdentifier(PHYSICAL_ROW_ID_ALIAS)}, ${columnsSql}
+              FROM ${tableSql} AT (VERSION => ${snapshotId})
+              ${physicalCursor === null ? "" : `WHERE rowid > ${physicalCursor}`}
+              ORDER BY rowid LIMIT ${pageRows}
+              ${physicalCursor === null && offset > 0 ? `OFFSET ${offset}` : ""}`
+          : `${baseSql}${orderSql} LIMIT ${pageRows} OFFSET ${offset}`
+        const rows = await runtime.query(pageSql)
+        if (rows.length === 0) return
+
+        yield* this.normalizeRows(rows, selectedColumns)
+        offset += rows.length
+        if (remaining !== undefined) {
+          remaining -= rows.length
+          if (remaining === 0) return
+        }
+        if (canUsePhysicalRowId) {
+          physicalCursor = getBigIntLike(rows[rows.length - 1]!, PHYSICAL_ROW_ID_ALIAS)
+        }
+        if (rows.length < pageRows) return
+      }
     } finally {
       await lease.release()
     }

package/src/internal/ducklake-sql-executor.ts

@@ -15,13 +15,14 @@ import {
   applyDatasetRowsFromRelation,
   assertDatasetWriteMode,
 } from "./dataset-row-commit"
+import { findDatasetColumnTypeForDuckDbSql } from "./duckdb-column-types"
 import { getString } from "./duckdb-row"
 import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
 import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
 import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader"
 import type { DuckLakeWriteCoordinator } from "./ducklake-write-coordinator"
-import { duckDbTypeToDatasetColumnType } from "./schema"
+import { datasetColumnTypeToDuckDbSql } from "./schema"
 import { quoteIdentifier } from "./sql"
 import {
   type DuckLakeSqlTransformSourceRelation,
@@ -42,9 +43,20 @@ interface ApplySqlTransformInput {
   readonly target: DatasetDefinition
   readonly mode: DatasetWriteMode
   readonly sql: string
+  readonly checkedCasts: readonly SqlTransformCheckedCast[]
   readonly previousRowCount?: number
 }
 
+interface SqlTransformResultColumn {
+  readonly name: string
+  readonly typeSql: string
+}
+
+interface SqlTransformCheckedCast {
+  readonly columnName: string
+  readonly targetTypeSql: string
+}
+
 /**
  * DuckDB-backed SQL transform executor for DuckLakeStorage.
  */
@@ -95,7 +107,7 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
         sql: input.sql,
       })
 
-      await this.assertResultSchemaMatchesTarget(runtime, target, sql)
+      const checkedCasts = await this.assertResultSchemaMatchesTarget(runtime, target, sql)
 
       return this.writes.commitDatasetVersionOnExclusiveRuntime(runtime, {
         dataset: target,
@@ -110,6 +122,7 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
             target,
             mode: input.mode,
             sql,
+            checkedCasts,
             previousRowCount: context.previousRowCount,
           }),
       })
@@ -199,12 +212,13 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
     runtime: DuckDbQueryRuntime,
     target: DatasetDefinition,
     sql: string
-  ): Promise<void> {
+  ): Promise<readonly SqlTransformCheckedCast[]> {
     const rows = await runtime.query(
       `DESCRIBE SELECT * FROM (${sql}) AS sixb_sql_transform_result_schema`
     )
     const actualColumns = rows.map((row) => resultColumnFromDescribeRow(row))
     const expectedColumns = target.schema.columns
+    const checkedCasts: SqlTransformCheckedCast[] = []
 
     if (actualColumns.length !== expectedColumns.length) {
       throwResultSchemaMismatch(
@@ -224,13 +238,26 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
         )
       }
 
-      if (actual.type !== expected.type) {
-        throwResultSchemaMismatch(
-          target.id,
-          `column '${expected.name}' should have type '${expected.type}', got '${actual.type}'`
-        )
+      const actualDatasetType = findDatasetColumnTypeForDuckDbSql(actual.typeSql)
+      if (actualDatasetType === expected.type) {
+        continue
       }
+
+      if (isCheckedInt64Cast(actual.typeSql, expected.type)) {
+        checkedCasts.push({
+          columnName: expected.name,
+          targetTypeSql: datasetColumnTypeToDuckDbSql(expected.type),
+        })
+        continue
+      }
+
+      throwResultSchemaMismatch(
+        target.id,
+        `column '${expected.name}' should have type '${expected.type}', got '${actualDatasetType ?? actual.typeSql}'`
+      )
     }
+
+    return checkedCasts
   }
 
   private async applySqlTransform(input: ApplySqlTransformInput): Promise<ApplyDatasetRowsResult> {
@@ -243,6 +270,14 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
 
     let result: ApplyDatasetRowsResult
     try {
+      for (const checkedCast of input.checkedCasts) {
+        await input.runtime.run(
+          `ALTER TABLE ${tempTable} ALTER COLUMN ${quoteIdentifier(
+            checkedCast.columnName
+          )} SET DATA TYPE ${checkedCast.targetTypeSql}`
+        )
+      }
+
       result = await applyDatasetRowsFromRelation({
         options: this.options,
         runtime: input.runtime,
@@ -280,11 +315,19 @@ function buildPreviewSql(sql: string, limit: number): string {
 
 function resultColumnFromDescribeRow(
   row: Readonly<Record<string, unknown>>
-): DatasetColumnDefinition {
+): SqlTransformResultColumn {
   const name = getString(row, "column_name")
-  const type = duckDbTypeToDatasetColumnType(getString(row, "column_type"))
+  const typeSql = getString(row, "column_type")
+
+  return { name, typeSql }
+}
 
-  return { name, type }
+function isCheckedInt64Cast(
+  actualTypeSql: string,
+  expectedType: DatasetColumnDefinition["type"]
+): boolean {
+  const normalizedType = actualTypeSql.trim().toUpperCase()
+  return expectedType === "int64" && (normalizedType === "HUGEINT" || normalizedType === "INT128")
 }
 
 function normalizePreviewRow(row: Readonly<Record<string, unknown>>): DatasetRow {

package/src/internal/ducklake-write-coordinator.ts

@@ -27,7 +27,14 @@ import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
 import { createDuckLakeMergeSession, type DuckLakeCommitMergeInput } from "./ducklake-merge-session"
 import type { DuckLakeSnapshotReader, DuckLakeVersionSummary } from "./ducklake-snapshot-reader"
 import { createDuckLakeWriteSession, type DuckLakeCommitWriteInput } from "./ducklake-write-session"
-import { duckLakeAlias, duckLakeMetadataTableName, quoteIdentifier, quoteSqlString } from "./sql"
+import { encodeDatasetTableName } from "./names"
+import {
+  duckLakeAlias,
+  duckLakeMetadataTableName,
+  qualifiedTableName,
+  quoteIdentifier,
+  quoteSqlString,
+} from "./sql"
 import { parseCommitMetadata, type SixbCommitMetadata } from "./versions"
 
 export interface DuckLakeCommitDatasetVersionInput {
@@ -325,6 +332,7 @@ export class DuckLakeWriteCoordinator {
         previousRowCount: latestVersion?.rowCount,
         validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns,
       })
+      await this.ensureInitialNoOpHasSnapshot(input, changeResult, latestVersion)
       const commitId = randomUUID()
       await this.setCommitMetadata(
         input,
@@ -479,6 +487,32 @@ export class DuckLakeWriteCoordinator {
     )
   }
 
+  private async ensureInitialNoOpHasSnapshot(
+    input: DuckLakeCommitVersionOutcomeRuntimeInput,
+    changeResult: ApplyDatasetRowsResult,
+    knownLatestVersion: DuckLakeVersionSummary | null
+  ): Promise<void> {
+    if (changeResult.dataChangeExpected || input.allowInitialNoOp) {
+      return
+    }
+
+    const latestVersion =
+      knownLatestVersion ??
+      (await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset))
+    if (latestVersion) {
+      return
+    }
+
+    // DuckLake 1.5.2 does not create a snapshot for a transaction whose data and metadata are
+    // unchanged, and set_commit_message alone cannot force one. Reapplying the current table
+    // comment is an idempotent metadata write: it leaves the visible DatasetDefinition unchanged
+    // while giving the first empty write a real DuckLake snapshot for time travel and lineage.
+    const table = qualifiedTableName(this.options, encodeDatasetTableName(input.dataset.id))
+    const descriptionSql =
+      input.dataset.description === undefined ? "NULL" : quoteSqlString(input.dataset.description)
+    await input.runtime.run(`COMMENT ON TABLE ${table} IS ${descriptionSql}`)
+  }
+
   private async versionForNoOpCommit(
     runtime: DuckDbQueryRuntime,
     definition: DatasetDefinition,

package/src/internal/ducklake-write-session.ts

@@ -106,6 +106,10 @@ class DuckLakeWriteSession implements LakeWriteSession {
         await this.appendBatchToStagingTable(batch, batchPrimaryKeys)
         batch = []
         batchPrimaryKeys = new Set()
+        // Native appender promises can settle through microtasks quickly enough to starve timers
+        // across a large in-memory iterable. Yield once per bounded batch so API traffic remains
+        // responsive without adding per-row scheduling overhead.
+        await Bun.sleep(0)
       }
     }