@sixb/ducklake 0.1.1 → 0.1.3

package/README.md

@@ -94,15 +94,22 @@ const report = await lakeStorage.runMaintenance({
 Reads, writes, SQL previews, and SQL transforms use the same DuckDB runtime and
 the same DuckLake PostgreSQL metadata pool. Work is serialized inside one
 `DuckLakeStorage` instance, so a queued DuckDB operation makes later operations
-wait. Only actual DuckDB work holds the runtime: batch appends, commits, SQL
-transforms, and metadata reads. A large streaming read can also make a later
-write wait until the stream is consumed or closed.
+wait. Only actual DuckDB work holds the runtime: bounded read pages, batch
+appends, commits, SQL transforms, and metadata reads.
+
+Large `readRows(...)` calls materialize bounded pages, using a physical-row keyset when available,
+and release the DuckDB queue between pages. This also applies when an explicit limit exceeds one
+page. A JavaScript pipeline can therefore stream one DuckLake dataset directly into another on the
+same provider without buffering the complete input or deadlocking at the destination appender's
+first flush. The read remains pinned to one immutable snapshot and keeps its attachment lease until
+the iterator completes or closes.
 
 A write does not hold the runtime while its source iterable is producing rows.
 `writeRows(...)` validates and stages rows in bounded in-memory batches, then
 takes a queue slot only to flush each batch into the staging table. Slow external
 reads (pagination, APIs, SFTP, retries) run outside the queue, so other reads and
-write batches can interleave between a write's batches.
+write batches can interleave between a write's batches. Large in-memory writes
+also yield to the event loop between batches.
 
 ```txt
 same provider instance:
@@ -263,6 +270,10 @@ new rows to the existing target. Omitting a source `versionId` resolves the late
 committed version at execution time; providing a `versionId` pins that source for
 repeatable transforms.
 
+DuckDB promotes `sum(BIGINT)` results to `HUGEINT`. When the matching target column is a Sixb
+`int64`, the executor applies a checked conversion back to `BIGINT` before comparing or writing
+rows. Values outside the signed 64-bit range fail the transform instead of being truncated.
+
 ## Definition Updates
 
 DuckLake supports schema evolution, and this provider exposes a small safe

package/dist/index.js

@@ -1784,6 +1784,9 @@ function isValidatedPrimaryKeyColumns(value) {
 }
 
 // src/internal/ducklake-row-reader.ts
+var READ_PAGE_ROWS = 5000;
+var PHYSICAL_ROW_ID_ALIAS = "__sixb_physical_row_id";
+
 class DuckLakeRowReader {
   options;
   connections;
@@ -1803,16 +1806,43 @@ class DuckLakeRowReader {
         throw new LakeStorageError14(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`);
       }
       const version = await this.resolveVersion(runtime, tableRef, input.versionId);
-      const snapshotId = parseVersionId(version.versionId);
       const selectedColumns = this.resolveReadColumns(tableRef.datasetId, version.schema, input.columns);
+      const snapshotId = parseVersionId(version.versionId);
       const columnsSql = selectedColumns.map((column) => quoteIdentifier(column.name)).join(", ");
       const tableSql = qualifiedTableName(this.options, tableRef.tableName);
-      const orderSql = version.schema.columns.some((column) => column.name.toLowerCase() === "rowid") ? "" : " ORDER BY rowid";
-      const limitSql = input.limit === undefined ? "" : ` LIMIT ${Math.max(0, Math.trunc(input.limit))}`;
-      const offsetSql = input.offset === undefined ? "" : ` OFFSET ${Math.max(0, Math.trunc(input.offset))}`;
-      const sql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})${orderSql}${limitSql}${offsetSql}`;
-      const rows = input.limit === undefined ? runtime.streamRows(sql) : await runtime.query(sql);
-      yield* this.normalizeRows(rows, selectedColumns);
+      const schemaNames = new Set(version.schema.columns.map((column) => column.name.toLowerCase()));
+      const canUsePhysicalRowId = !schemaNames.has("rowid") && !schemaNames.has(PHYSICAL_ROW_ID_ALIAS.toLowerCase());
+      const orderSql = canUsePhysicalRowId ? " ORDER BY rowid" : "";
+      const baseSql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})`;
+      const requestedOffset = Math.max(0, Math.trunc(input.offset ?? 0));
+      let remaining = input.limit === undefined ? undefined : Math.max(0, Math.trunc(input.limit));
+      if (remaining === 0)
+        return;
+      let offset = requestedOffset;
+      let physicalCursor = null;
+      while (true) {
+        const pageRows = remaining === undefined ? READ_PAGE_ROWS : Math.min(READ_PAGE_ROWS, remaining);
+        const pageSql = canUsePhysicalRowId ? `SELECT rowid AS ${quoteIdentifier(PHYSICAL_ROW_ID_ALIAS)}, ${columnsSql}
+              FROM ${tableSql} AT (VERSION => ${snapshotId})
+              ${physicalCursor === null ? "" : `WHERE rowid > ${physicalCursor}`}
+              ORDER BY rowid LIMIT ${pageRows}
+              ${physicalCursor === null && offset > 0 ? `OFFSET ${offset}` : ""}` : `${baseSql}${orderSql} LIMIT ${pageRows} OFFSET ${offset}`;
+        const rows = await runtime.query(pageSql);
+        if (rows.length === 0)
+          return;
+        yield* this.normalizeRows(rows, selectedColumns);
+        offset += rows.length;
+        if (remaining !== undefined) {
+          remaining -= rows.length;
+          if (remaining === 0)
+            return;
+        }
+        if (canUsePhysicalRowId) {
+          physicalCursor = getBigIntLike(rows[rows.length - 1], PHYSICAL_ROW_ID_ALIAS);
+        }
+        if (rows.length < pageRows)
+          return;
+      }
     } finally {
       await lease.release();
     }
@@ -2606,7 +2636,7 @@ class DuckLakeSqlExecutor {
         sources: sources.relations,
         sql: input.sql
       });
-      await this.assertResultSchemaMatchesTarget(runtime, target, sql);
+      const checkedCasts = await this.assertResultSchemaMatchesTarget(runtime, target, sql);
       return this.writes.commitDatasetVersionOnExclusiveRuntime(runtime, {
         dataset: target,
         mode: input.mode,
@@ -2619,6 +2649,7 @@ class DuckLakeSqlExecutor {
           target,
           mode: input.mode,
           sql,
+          checkedCasts,
           previousRowCount: context.previousRowCount
         })
       });
@@ -2678,6 +2709,7 @@ class DuckLakeSqlExecutor {
     const rows = await runtime.query(`DESCRIBE SELECT * FROM (${sql}) AS sixb_sql_transform_result_schema`);
     const actualColumns = rows.map((row) => resultColumnFromDescribeRow(row));
     const expectedColumns = target.schema.columns;
+    const checkedCasts = [];
     if (actualColumns.length !== expectedColumns.length) {
       throwResultSchemaMismatch(target.id, `expected ${expectedColumns.length} columns, got ${actualColumns.length}`);
     }
@@ -2687,10 +2719,20 @@ class DuckLakeSqlExecutor {
       if (actual.name !== expected.name) {
         throwResultSchemaMismatch(target.id, `column ${index + 1} should be '${expected.name}', got '${actual.name}'`);
       }
-      if (actual.type !== expected.type) {
-        throwResultSchemaMismatch(target.id, `column '${expected.name}' should have type '${expected.type}', got '${actual.type}'`);
+      const actualDatasetType = findDatasetColumnTypeForDuckDbSql(actual.typeSql);
+      if (actualDatasetType === expected.type) {
+        continue;
       }
+      if (isCheckedInt64Cast(actual.typeSql, expected.type)) {
+        checkedCasts.push({
+          columnName: expected.name,
+          targetTypeSql: datasetColumnTypeToDuckDbSql(expected.type)
+        });
+        continue;
+      }
+      throwResultSchemaMismatch(target.id, `column '${expected.name}' should have type '${expected.type}', got '${actualDatasetType ?? actual.typeSql}'`);
     }
+    return checkedCasts;
   }
   async applySqlTransform(input) {
     const tempTableName = `sixb_sql_transform_${randomUUID3().replaceAll("-", "")}`;
@@ -2698,6 +2740,9 @@ class DuckLakeSqlExecutor {
     await input.runtime.run(`CREATE TEMP TABLE ${tempTable} AS SELECT * FROM (${input.sql}) AS sixb_sql_transform_result`);
     let result;
     try {
+      for (const checkedCast of input.checkedCasts) {
+        await input.runtime.run(`ALTER TABLE ${tempTable} ALTER COLUMN ${quoteIdentifier(checkedCast.columnName)} SET DATA TYPE ${checkedCast.targetTypeSql}`);
+      }
       result = await applyDatasetRowsFromRelation({
         options: this.options,
         runtime: input.runtime,
@@ -2727,8 +2772,12 @@ function buildPreviewSql(sql, limit) {
 }
 function resultColumnFromDescribeRow(row) {
   const name = getString(row, "column_name");
-  const type = duckDbTypeToDatasetColumnType(getString(row, "column_type"));
-  return { name, type };
+  const typeSql = getString(row, "column_type");
+  return { name, typeSql };
+}
+function isCheckedInt64Cast(actualTypeSql, expectedType) {
+  const normalizedType = actualTypeSql.trim().toUpperCase();
+  return expectedType === "int64" && (normalizedType === "HUGEINT" || normalizedType === "INT128");
 }
 function normalizePreviewRow(row) {
   const normalized = {};
@@ -3049,6 +3098,7 @@ class DuckLakeMergeSession {
       if (batch.length >= MERGE_CHANGE_BATCH_SIZE) {
         await this.appendBatchToStagingTable(batch);
         batch = [];
+        await Bun.sleep(0);
       }
     }
     if (batch.length > 0) {
@@ -3164,6 +3214,7 @@ class DuckLakeWriteSession {
         await this.appendBatchToStagingTable(batch, batchPrimaryKeys);
         batch = [];
         batchPrimaryKeys = new Set;
+        await Bun.sleep(0);
       }
     }
     if (batch.length > 0) {
@@ -3400,6 +3451,7 @@ class DuckLakeWriteCoordinator {
         previousRowCount: latestVersion?.rowCount,
         validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns
       });
+      await this.ensureInitialNoOpHasSnapshot(input, changeResult, latestVersion);
       const commitId = randomUUID6();
       await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion));
       await input.runtime.run("COMMIT");
@@ -3481,6 +3533,18 @@ class DuckLakeWriteCoordinator {
     };
     await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
   }
+  async ensureInitialNoOpHasSnapshot(input, changeResult, knownLatestVersion) {
+    if (changeResult.dataChangeExpected || input.allowInitialNoOp) {
+      return;
+    }
+    const latestVersion = knownLatestVersion ?? await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
+    if (latestVersion) {
+      return;
+    }
+    const table = qualifiedTableName(this.options, encodeDatasetTableName(input.dataset.id));
+    const descriptionSql = input.dataset.description === undefined ? "NULL" : quoteSqlString(input.dataset.description);
+    await input.runtime.run(`COMMENT ON TABLE ${table} IS ${descriptionSql}`);
+  }
   async versionForNoOpCommit(runtime, definition, allowInitialNoOp) {
     const latestVersion = await this.snapshots.getLatestVersionForDefinition(runtime, definition);
     if (latestVersion) {
@@ -3664,4 +3728,4 @@ export {
   DuckLakeStorage
 };
 
-//# debugId=8AEF2C0D635BEB8A64756E2164756E21
+//# debugId=85151472B319A47E64756E2164756E21